From a4d9d829b2e2736c933b677dd8c19731249e76d3 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Thu, 4 Sep 2025 23:29:42 +0200 Subject: [PATCH 01/21] download client side and bulk download --- src/apps/api/views/submissions.py | 73 +++++++++++++++-- src/apps/competitions/tasks.py | 82 +++++++++---------- src/static/js/ours/client.js | 55 ++++++++----- .../detail/submission_manager.tag | 6 +- 4 files changed, 142 insertions(+), 74 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index afe14fb36..6549263e6 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -349,6 +349,7 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) + @action(detail=False, methods=['get']) def download_many(self, request): """ @@ -363,12 +364,15 @@ def download_many(self, request): # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", - "phase__competition", - "phase__competition__created_by", - ).prefetch_related("phase__competition__collaborators") + "phase", + "data", + ) + # .only("id","owner", "data__data_file") + # .prefetch_related("phase__competition__collaborators") if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -389,12 +393,63 @@ def download_many(self, request): "You do not have permission to download one or more of the requested submissions" ) - # Download - from competitions.tasks import stream_batch_download - in_memory_zip = stream_batch_download(pks) - response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') - response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' - return response + files = [] + for sub in submissions: + file_path = sub.data.data_file.name.split('/')[-1] + short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" + + url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] + files.append({"name": short_name, "url": url}) + + return Response(files) + + + # @action(detail=False, methods=['get']) + # def download_many(self, request): + # """ + # Download a ZIP containing several submissions. + # """ + # pks = request.query_params.get('pks') + # if pks: + # pks = json.loads(pks) # Convert JSON string to list + # else: + # return Response({"error": "`pks` query parameter is required"}, status=400) + + # # Get submissions + # submissions = Submission.objects.filter(pk__in=pks).select_related( + # "owner", + # "phase__competition", + # "phase__competition__created_by", + # ).prefetch_related("phase__competition__collaborators") + # if submissions.count() != len(pks): + # return Response({"error": "One or more submission IDs are invalid"}, status=404) + + # # Check permissions + # if not request.user.is_authenticated: + # raise PermissionDenied("You must be logged in to download submissions") + # # Allow admins + # if request.user.is_superuser or request.user.is_staff: + # allowed = True + # else: + # # Build one Q object for "owner OR organizer" + # organiser_q = ( + # Q(phase__competition__created_by=request.user) | + # Q(phase__competition__collaborators=request.user) + # ) + # # Submissions that violate the rule + # disallowed = submissions.exclude(Q(owner=request.user) | organiser_q) + # allowed = not disallowed.exists() + # if not allowed: + # raise PermissionDenied( + # "You do not have permission to download one or more of the requested submissions" + # ) + + # # Download + # from competitions.tasks import stream_batch_download + # in_memory_zip = stream_batch_download(pks) + # response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') + # response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' + # return response @action(detail=True, methods=('GET',)) def get_details(self, request, pk): diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 9b685d1d7..9dd0e978f 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -280,47 +280,47 @@ def send_child_id(submission, child_id): }) -def retrieve_data(url, data=None): - with closing(urlopen(url, data)) as fp: - headers = fp.info() - - bs = 1024 * 8 - size = -1 - read = 0 - if "content-length" in headers: - size = int(headers["Content-Length"]) - - while True: - block = fp.read(bs) - if not block: - break - read += len(block) - yield(block) - - if size >= 0 and read < size: - raise ContentTooShortError( - "retrieval incomplete: got only %i out of %i bytes" - % (read, size)) - - -def zip_generator(submission_pks): - in_memory_zip = BytesIO() - with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: - for submission_id in submission_pks: - submission = Submission.objects.get(id=submission_id) - short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] - url = make_url_sassy(path=submission.data.data_file.name) - for block in retrieve_data(url): - zip_file.writestr(short_name, block) - - in_memory_zip.seek(0) - - return in_memory_zip - - -@app.task(queue='site-worker', soft_time_limit=60 * 60) -def stream_batch_download(submission_pks): - return zip_generator(submission_pks) +# def retrieve_data(url, data=None): +# with closing(urlopen(url, data)) as fp: +# headers = fp.info() + +# bs = 1024 * 8 +# size = -1 +# read = 0 +# if "content-length" in headers: +# size = int(headers["Content-Length"]) + +# while True: +# block = fp.read(bs) +# if not block: +# break +# read += len(block) +# yield(block) + +# if size >= 0 and read < size: +# raise ContentTooShortError( +# "retrieval incomplete: got only %i out of %i bytes" +# % (read, size)) + + +# def zip_generator(submission_pks): +# in_memory_zip = BytesIO() +# with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: +# for submission_id in submission_pks: +# submission = Submission.objects.get(id=submission_id) +# short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] +# url = make_url_sassy(path=submission.data.data_file.name) +# for block in retrieve_data(url): +# zip_file.writestr(short_name, block) + +# in_memory_zip.seek(0) + +# return in_memory_zip + + +# @app.task(queue='site-worker', soft_time_limit=60 * 60) +# def stream_batch_download(submission_pks): +# return zip_generator(submission_pks) @app.task(queue='site-worker', soft_time_limit=60) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 410ec34cf..963b11057 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -128,29 +128,42 @@ CODALAB.api = { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, download_many_submissions: function (pks) { - console.log('Request bulk'); const params = new URLSearchParams({ pks: JSON.stringify(pks) }); const url = `${URLS.API}submissions/download_many/?${params}`; - return fetch(url, { - method: 'GET', - headers: { - 'Content-Type': 'application/json' - } - }).then(response => { - if (!response.ok) { - throw new Error('Network response was not ok ' + response.statusText); - } - return response.blob(); - }).then(blob => { - const link = document.createElement('a'); - link.href = window.URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }).catch(error => { - console.error('Error downloading submissions:', error); - }); + // return CODALAB.api.request('GET', url) + CODALAB.api.request('GET', url) + .done(function(files) { + // files is already parsed JSON array + console.log("Files returned by server:", files); + + // Iterate files + files.forEach(file => { + console.log(file.name, file.url); + }); + // return files + + const zip = new JSZip(); + const fetchFiles = files.map(async file => { + const response = await fetch(file.url); + const blob = await response.blob(); + zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + }); + + Promise.all(fetchFiles).then(() => { + zip.generateAsync({ type: 'blob' }).then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }); + }); + + }) + .fail(function(err) { + console.error("Error downloading submissions:", err); + }); }, /*--------------------------------------------------------------------- diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 74b746cc6..03e0c1c9e 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -571,9 +571,9 @@ self.bulk_download = function () { CODALAB.api.download_many_submissions(self.checked_submissions) - .catch(function (error) { - console.error('Error:', error); - }); + #.catch(function (error) { + # console.error('Error:', error); + #}); } From 5daa6f9528d2abb95de55d02755a50e779019bd7 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Mon, 8 Sep 2025 16:44:57 +0200 Subject: [PATCH 02/21] working bulk download --- src/static/js/ours/client.js | 58 +++++------ .../detail/submission_manager.tag | 97 +++++++++++++++---- src/templates/base.html | 1 + 3 files changed, 107 insertions(+), 49 deletions(-) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 963b11057..a1f1ddebb 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -131,39 +131,39 @@ CODALAB.api = { const params = new URLSearchParams({ pks: JSON.stringify(pks) }); const url = `${URLS.API}submissions/download_many/?${params}`; // return CODALAB.api.request('GET', url) - CODALAB.api.request('GET', url) - .done(function(files) { - // files is already parsed JSON array - console.log("Files returned by server:", files); + return CODALAB.api.request('GET', url) + // .done(function(files) { + // // files is already parsed JSON array + // console.log("Files returned by server:", files); - // Iterate files - files.forEach(file => { - console.log(file.name, file.url); - }); - // return files + // // Iterate files + // files.forEach(file => { + // console.log(file.name, file.url); + // }); + // // return files - const zip = new JSZip(); - const fetchFiles = files.map(async file => { - const response = await fetch(file.url); - const blob = await response.blob(); - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - }); + // const zip = new JSZip(); + // const fetchFiles = files.map(async file => { + // const response = await fetch(file.url); + // const blob = await response.blob(); + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + // }); - Promise.all(fetchFiles).then(() => { - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); - link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }); - }); + // Promise.all(fetchFiles).then(() => { + // zip.generateAsync({ type: 'blob' }).then(blob => { + // const link = document.createElement('a'); + // link.href = URL.createObjectURL(blob); + // link.download = 'bulk_submissions.zip'; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + // }); + // }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // }); }, /*--------------------------------------------------------------------- diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 03e0c1c9e..79d894bc3 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -1,29 +1,36 @@ -
-
+
+
+
Rerun all submissions per phase
Select a phase
-
- { phase.name } +
+ { phase.name }
-
- +
+ + Download as CSV - + - + - - + +
+
@@ -568,15 +575,65 @@ CODALAB.events.trigger('submission_clicked') } - self.bulk_download = function () { - CODALAB.api.download_many_submissions(self.checked_submissions) - #.catch(function (error) { - # console.error('Error:', error); - #}); - } + const statusBox = document.getElementById('downloadStatus'); + const progressEl = document.getElementById('downloadProgress'); + const textEl = document.getElementById('progressText'); + statusBox.style.display = "flex"; + // statusBox.style.display = "inline"; + + progressEl.style.display="flex"; + progressEl.value = 0; + textEl.textContent = "Preparing download..."; + + CODALAB.api.download_many_submissions(self.checked_submissions) + .done(function(files) { + console.log("Files returned by server:", files); + + const zip = new JSZip(); + const total = files.length; + let completed = 0; + + const fetchFiles = files.map(async file => { + const response = await fetch(file.url); + const blob = await response.blob(); + + zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + + // Update progress + completed++; + const percent = Math.round((completed / total) * 100); + progressEl.value = percent; + // progressBar.progress({ percent: percent }); + textEl.textContent = `${completed} / ${total} files (${percent}%)`; + }); + + Promise.all(fetchFiles).then(() => { + textEl.textContent = "Generating bundle"; + progressEl.style.display="none"; + + zip.generateAsync({ type: 'blob' }).then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + textEl.textContent = "Download ready!"; + setTimeout(() => { + statusBox.style.display = "none"; + }, 3000); + }); + }); + }) + .fail(function(err) { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + }); + }; self.submission_handling = function () { // console.log(self.checked_submissions) diff --git a/src/templates/base.html b/src/templates/base.html index 30584b35a..014a7c63d 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -256,6 +256,7 @@

CodaBench

+ From eb60cde2256d64f0a86e8fbd35bb61bc2a301c44 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Mon, 8 Sep 2025 17:19:50 +0200 Subject: [PATCH 03/21] cleaned --- src/apps/api/views/submissions.py | 50 +------------------------------ src/apps/competitions/tasks.py | 43 -------------------------- src/static/js/ours/client.js | 33 -------------------- 3 files changed, 1 insertion(+), 125 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 6549263e6..68b01b7ae 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -365,7 +365,7 @@ def download_many(self, request): submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", - "data", + "data" ) # .only("id","owner", "data__data_file") # .prefetch_related("phase__competition__collaborators") @@ -403,54 +403,6 @@ def download_many(self, request): return Response(files) - - # @action(detail=False, methods=['get']) - # def download_many(self, request): - # """ - # Download a ZIP containing several submissions. - # """ - # pks = request.query_params.get('pks') - # if pks: - # pks = json.loads(pks) # Convert JSON string to list - # else: - # return Response({"error": "`pks` query parameter is required"}, status=400) - - # # Get submissions - # submissions = Submission.objects.filter(pk__in=pks).select_related( - # "owner", - # "phase__competition", - # "phase__competition__created_by", - # ).prefetch_related("phase__competition__collaborators") - # if submissions.count() != len(pks): - # return Response({"error": "One or more submission IDs are invalid"}, status=404) - - # # Check permissions - # if not request.user.is_authenticated: - # raise PermissionDenied("You must be logged in to download submissions") - # # Allow admins - # if request.user.is_superuser or request.user.is_staff: - # allowed = True - # else: - # # Build one Q object for "owner OR organizer" - # organiser_q = ( - # Q(phase__competition__created_by=request.user) | - # Q(phase__competition__collaborators=request.user) - # ) - # # Submissions that violate the rule - # disallowed = submissions.exclude(Q(owner=request.user) | organiser_q) - # allowed = not disallowed.exists() - # if not allowed: - # raise PermissionDenied( - # "You do not have permission to download one or more of the requested submissions" - # ) - - # # Download - # from competitions.tasks import stream_batch_download - # in_memory_zip = stream_batch_download(pks) - # response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') - # response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' - # return response - @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 9dd0e978f..124711707 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -280,49 +280,6 @@ def send_child_id(submission, child_id): }) -# def retrieve_data(url, data=None): -# with closing(urlopen(url, data)) as fp: -# headers = fp.info() - -# bs = 1024 * 8 -# size = -1 -# read = 0 -# if "content-length" in headers: -# size = int(headers["Content-Length"]) - -# while True: -# block = fp.read(bs) -# if not block: -# break -# read += len(block) -# yield(block) - -# if size >= 0 and read < size: -# raise ContentTooShortError( -# "retrieval incomplete: got only %i out of %i bytes" -# % (read, size)) - - -# def zip_generator(submission_pks): -# in_memory_zip = BytesIO() -# with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: -# for submission_id in submission_pks: -# submission = Submission.objects.get(id=submission_id) -# short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] -# url = make_url_sassy(path=submission.data.data_file.name) -# for block in retrieve_data(url): -# zip_file.writestr(short_name, block) - -# in_memory_zip.seek(0) - -# return in_memory_zip - - -# @app.task(queue='site-worker', soft_time_limit=60 * 60) -# def stream_batch_download(submission_pks): -# return zip_generator(submission_pks) - - @app.task(queue='site-worker', soft_time_limit=60) def _run_submission(submission_pk, task_pks=None, is_scoring=False): """This function is wrapped so that when we run tests we can run this function not diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index a1f1ddebb..03746945b 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -130,40 +130,7 @@ CODALAB.api = { download_many_submissions: function (pks) { const params = new URLSearchParams({ pks: JSON.stringify(pks) }); const url = `${URLS.API}submissions/download_many/?${params}`; - // return CODALAB.api.request('GET', url) return CODALAB.api.request('GET', url) - // .done(function(files) { - // // files is already parsed JSON array - // console.log("Files returned by server:", files); - - // // Iterate files - // files.forEach(file => { - // console.log(file.name, file.url); - // }); - // // return files - - // const zip = new JSZip(); - // const fetchFiles = files.map(async file => { - // const response = await fetch(file.url); - // const blob = await response.blob(); - // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - // }); - - // Promise.all(fetchFiles).then(() => { - // zip.generateAsync({ type: 'blob' }).then(blob => { - // const link = document.createElement('a'); - // link.href = URL.createObjectURL(blob); - // link.download = 'bulk_submissions.zip'; - // document.body.appendChild(link); - // link.click(); - // document.body.removeChild(link); - // }); - // }); - - // }) - // .fail(function(err) { - // console.error("Error downloading submissions:", err); - // }); }, /*--------------------------------------------------------------------- From ec29a376e24672f5fc90b8972622949ec1f3d6e1 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Wed, 17 Sep 2025 15:29:58 +0200 Subject: [PATCH 04/21] gestion of faulty submission --- src/apps/api/views/submissions.py | 5 +- .../detail/submission_manager.tag | 150 +++++++++++++----- 2 files changed, 108 insertions(+), 47 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 68b01b7ae..25a88f360 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -349,7 +349,6 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - @action(detail=False, methods=['get']) def download_many(self, request): """ @@ -372,7 +371,7 @@ def download_many(self, request): if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) - # NH : should should create a function for this ? + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -393,7 +392,7 @@ def download_many(self, request): "You do not have permission to download one or more of the requested submissions" ) - files = [] + files = [] for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 79d894bc3..80f0cc5ef 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -574,7 +574,8 @@ .modal('show') CODALAB.events.trigger('submission_clicked') } - + + self.bulk_download = function () { const statusBox = document.getElementById('downloadStatus'); const progressEl = document.getElementById('downloadProgress'); @@ -589,50 +590,111 @@ textEl.textContent = "Preparing download..."; CODALAB.api.download_many_submissions(self.checked_submissions) - .done(function(files) { - console.log("Files returned by server:", files); - - const zip = new JSZip(); - const total = files.length; - let completed = 0; - - const fetchFiles = files.map(async file => { - const response = await fetch(file.url); - const blob = await response.blob(); - - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - - // Update progress - completed++; - const percent = Math.round((completed / total) * 100); - progressEl.value = percent; - // progressBar.progress({ percent: percent }); - textEl.textContent = `${completed} / ${total} files (${percent}%)`; - }); - - Promise.all(fetchFiles).then(() => { - textEl.textContent = "Generating bundle"; - progressEl.style.display="none"; - - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); - link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - - textEl.textContent = "Download ready!"; - setTimeout(() => { - statusBox.style.display = "none"; - }, 3000); - }); - }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - textEl.textContent = "Error downloading!"; + .done(function(files) { + console.log("Files returned by server:", files); + + const zip = new JSZip(); + const total = files.length; + let completed = 0; + const failed = []; + + const fetchFiles = files.map(async file => { + try { + const response = await fetch(file.url); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const blob = await response.blob(); + + zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + } catch (err) { + console.error(`Failed to fetch ${file.name}:`, err); + failed.push(file.name); + } finally { + // Update progress regardless of success/failure + completed++; + const percent = Math.floor((completed / total) * 100); + progressEl.value = percent; + textEl.textContent = `${completed} / ${total} files (${percent}%)`; + } }); + + Promise.allSettled(fetchFiles).then(() => { + textEl.textContent = "Generating bundle"; + progressEl.style.display = "none"; + + zip.generateAsync({ type: 'blob' }).then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + if (failed.length > 0) { + textEl.textContent = `Download complete, but ${failed.length} failed: ${failed.join(', ')}`; + } else { + textEl.textContent = "Download ready!"; + } + + setTimeout(() => { + statusBox.style.display = "none"; + }, 5000); + }); + }); + }) + .fail(function(err) { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + }); + + // CODALAB.api.download_many_submissions(self.checked_submissions) + // .done(function(files) { + // console.log("Files returned by server:", files); + + // const zip = new JSZip(); + // const total = files.length; + // let completed = 0; + + // const fetchFiles = files.map(async file => { + // const response = await fetch(file.url); + // const blob = await response.blob(); + + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + + // // Update progress + // completed++; + // const percent = Math.floor((completed / total) * 100); + // progressEl.value = percent; + // // progressBar.progress({ percent: percent }); + // textEl.textContent = `${completed} / ${total} files (${percent}%)`; + // }); + + // Promise.all(fetchFiles).then(() => { + // textEl.textContent = "Generating bundle"; + // progressEl.style.display="none"; + + // zip.generateAsync({ type: 'blob' }).then(blob => { + // const link = document.createElement('a'); + // link.href = URL.createObjectURL(blob); + // link.download = 'bulk_submissions.zip'; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + + // textEl.textContent = "Download ready!"; + // setTimeout(() => { + // statusBox.style.display = "none"; + // }, 3000); + // }); + // }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // textEl.textContent = "Error downloading!"; + // }); }; self.submission_handling = function () { From 6154fc774a54c5b81ef934577a22b8b840c00f3d Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Thu, 18 Sep 2025 15:02:46 +0200 Subject: [PATCH 05/21] changed to post instead of get and added failed.txt to bundle --- src/apps/api/views/submissions.py | 29 ++++++++++++------- src/static/js/ours/client.js | 21 +++++++++++--- .../detail/submission_manager.tag | 14 ++++++--- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 25a88f360..8e7cf3ec9 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -349,17 +349,26 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - @action(detail=False, methods=['get']) + # @action(detail=False, methods=['get']) + # def download_many(self, request): + # """ + # Download a ZIP containing several submissions. + # """ + # pks = request.query_params.get('pks') + # if pks: + # pks = json.loads(pks) # Convert JSON string to list + # else: + # return Response({"error": "`pks` query parameter is required"}, status=400) + @action(detail=False, methods=['POST']) def download_many(self, request): - """ - Download a ZIP containing several submissions. - """ - pks = request.query_params.get('pks') - if pks: - pks = json.loads(pks) # Convert JSON string to list - else: - return Response({"error": "`pks` query parameter is required"}, status=400) - + pks = request.data.get('pks') + if not pks: + return Response({"error": "`pks` field is required"}, status=400) + + # pks is already parsed as a list if JSON was sent properly + if not isinstance(pks, list): + return Response({"error": "`pks` must be a list"}, status=400) + # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 03746945b..b3a8291c1 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -128,11 +128,24 @@ CODALAB.api = { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, download_many_submissions: function (pks) { - const params = new URLSearchParams({ pks: JSON.stringify(pks) }); - const url = `${URLS.API}submissions/download_many/?${params}`; - return CODALAB.api.request('GET', url) + return CODALAB.api.request( + 'POST', + URLS.API + "submissions/download_many/", + { pks: pks } // body is JSON by convention + ); }, - + + // download_many_submissions: function (pks) { + // return CODALAB.api.request('POST', `${URLS.API}submissions/download_many/`, { + // body: JSON.stringify({ pks: pks }), + // headers: { + // "Content-Type": "application/json" + // } + // }); + // const params = new URLSearchParams({ pks: JSON.stringify(pks) }); + // const url = `${URLS.API}submissions/download_many/?${params}`; + // return CODALAB.api.request('GET', url) + // }, /*--------------------------------------------------------------------- Leaderboards ---------------------------------------------------------------------*/ diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 80f0cc5ef..96ba0151a 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -622,19 +622,25 @@ }); Promise.allSettled(fetchFiles).then(() => { + // If some files failed, include them as failed.txt inside the zip + if (failed.length > 0) { + const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + zip.file("failed.txt", failedContent); + } + textEl.textContent = "Generating bundle"; progressEl.style.display = "none"; - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); + zip.generateAsync({ type: "blob" }).then(blob => { + const link = document.createElement("a"); link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; + link.download = "bulk_submissions.zip"; document.body.appendChild(link); link.click(); document.body.removeChild(link); if (failed.length > 0) { - textEl.textContent = `Download complete, but ${failed.length} failed: ${failed.join(', ')}`; + textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; } else { textEl.textContent = "Download ready!"; } From 27357e296175e530adc482723eca776046ccdd59 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 19 Sep 2025 12:01:47 +0200 Subject: [PATCH 06/21] added limit on number of file to download simultaneously --- src/apps/api/views/submissions.py | 14 +- .../detail/submission_manager.tag | 278 ++++++++++++------ 2 files changed, 191 insertions(+), 101 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 8e7cf3ec9..4be8895d9 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -349,17 +349,9 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - # @action(detail=False, methods=['get']) - # def download_many(self, request): - # """ - # Download a ZIP containing several submissions. - # """ - # pks = request.query_params.get('pks') - # if pks: - # pks = json.loads(pks) # Convert JSON string to list - # else: - # return Response({"error": "`pks` query parameter is required"}, status=400) - @action(detail=False, methods=['POST']) + + + @action(detail=False, methods=('POST',)) def download_many(self, request): pks = request.data.get('pks') if not pks: diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 96ba0151a..148980fdc 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -575,22 +575,139 @@ CODALAB.events.trigger('submission_clicked') } - + + // self.bulk_download = function () { + // const statusBox = document.getElementById('downloadStatus'); + // const progressEl = document.getElementById('downloadProgress'); + // const textEl = document.getElementById('progressText'); + + // statusBox.style.display = "flex"; + // // statusBox.style.display = "inline"; + + + // progressEl.style.display="flex"; + // progressEl.value = 0; + // textEl.textContent = "Preparing download..."; + + // console.log("Files returned by server:", files); + + + // CODALAB.api.download_many_submissions(self.checked_submissions) + // .done( async function(files) { + // // .done( function(files) { + // console.log("Files returned by server:", files); + + // const zip = new JSZip(); + // const total = files.length; + // let completed = 0; + // const failed = []; + + // const fetchFiles = files.map(async file => { + // try { + // const response = await fetch(file.url); + + // if (!response.ok) { + // throw new Error(`HTTP ${response.status}`); + // } + + // const blob = await response.blob(); + + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + // } catch (err) { + // console.error(`Failed to fetch ${file.name}:`, err); + // failed.push(file.name); + // } finally { + // // Update progress regardless of success/failure + // completed++; + // const percent = Math.floor((completed / total) * 100); + // progressEl.value = percent; + // textEl.textContent = `${completed} / ${total} files (${percent}%)`; + // } + // }); + + // Promise.allSettled(fetchFiles).then(() => { + // // If some files failed, include them as failed.txt inside the zip + // if (failed.length > 0) { + // const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + // zip.file("failed.txt", failedContent); + // } + + + // textEl.textContent = "Generating bundle"; + // progressEl.style.display = "none"; + + // zip.generateAsync({ type: "blob" }).then(blob => { + // const link = document.createElement("a"); + // link.href = URL.createObjectURL(blob); + // link.download = "bulk_submissions.zip"; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + + // if (failed.length > 0) { + // textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + // } else { + // textEl.textContent = "Download ready!"; + // } + + // setTimeout(() => { + // statusBox.style.display = "none"; + // }, 5000); + // }); + // }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // textEl.textContent = "Error downloading!"; + // }); + // }; + self.bulk_download = function () { const statusBox = document.getElementById('downloadStatus'); const progressEl = document.getElementById('downloadProgress'); const textEl = document.getElementById('progressText'); statusBox.style.display = "flex"; - // statusBox.style.display = "inline"; - - - progressEl.style.display="flex"; + progressEl.style.display = "flex"; progressEl.value = 0; textEl.textContent = "Preparing download..."; - CODALAB.api.download_many_submissions(self.checked_submissions) - .done(function(files) { + // Kick the API request + const req = CODALAB.api.download_many_submissions(self.checked_submissions); + + // Common error handler + const handleError = (err) => { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Success handler (async because we await inside) + const handleSuccess = async (resp) => { + // Normalize response -> files array + let files = resp; + if (resp && typeof resp === 'object' && !Array.isArray(resp)) { + // common shapes: { files: [...] } or { data: [...] } or direct array + if (Array.isArray(resp.files)) files = resp.files; + else if (Array.isArray(resp.data)) files = resp.data; + else if (Array.isArray(resp.results)) files = resp.results; + else if (Array.isArray(resp)) files = resp; + else { + // if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg + if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0]; + else { + console.warn("Unexpected response shape from download_many_submissions:", resp); + files = []; + } + } + } + + if (!Array.isArray(files) || files.length === 0) { + textEl.textContent = "No files to download"; + setTimeout(() => { statusBox.style.display = "none"; }, 3000); + return; + } + console.log("Files returned by server:", files); const zip = new JSZip(); @@ -598,109 +715,90 @@ let completed = 0; const failed = []; - const fetchFiles = files.map(async file => { + // concurrency limit (tune this; 5-10 is usually a good start) + const limit = 5; + const queue = files.slice(); // clone + const running = []; + + // worker to fetch one file + async function worker(file) { try { const response = await fetch(file.url); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); - } - + if (!response.ok) throw new Error(`HTTP ${response.status}`); const blob = await response.blob(); - - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + zip.file(file.name.replace(/[:/\\]/g, "_"), blob); } catch (err) { console.error(`Failed to fetch ${file.name}:`, err); - failed.push(file.name); + failed.push(`${file.name} (${err.message || 'fetch error'})`); } finally { - // Update progress regardless of success/failure completed++; const percent = Math.floor((completed / total) * 100); progressEl.value = percent; textEl.textContent = `${completed} / ${total} files (${percent}%)`; } - }); + } + + // run queue with limited concurrency + while (queue.length > 0) { + while (running.length < limit && queue.length > 0) { + const file = queue.shift(); + const p = worker(file).then(() => { + // remove finished promise from running + const idx = running.indexOf(p); + if (idx !== -1) running.splice(idx, 1); + }); + running.push(p); + } + // Wait for at least one running promise to finish + if (running.length > 0) { + await Promise.race(running); + } + } + // wait for remaining ones + await Promise.all(running); - Promise.allSettled(fetchFiles).then(() => { - // If some files failed, include them as failed.txt inside the zip + // Add failed.txt if necessary if (failed.length > 0) { - const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; - zip.file("failed.txt", failedContent); + const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + zip.file("failed.txt", failedContent); } textEl.textContent = "Generating bundle"; progressEl.style.display = "none"; - zip.generateAsync({ type: "blob" }).then(blob => { - const link = document.createElement("a"); - link.href = URL.createObjectURL(blob); - link.download = "bulk_submissions.zip"; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); + const blob = await zip.generateAsync({ type: "blob" }); + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = "bulk_submissions.zip"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); - if (failed.length > 0) { - textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; - } else { - textEl.textContent = "Download ready!"; - } + if (failed.length > 0) { + textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + } else { + textEl.textContent = "Download ready!"; + } - setTimeout(() => { - statusBox.style.display = "none"; - }, 5000); - }); - }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - textEl.textContent = "Error downloading!"; - }); - - // CODALAB.api.download_many_submissions(self.checked_submissions) - // .done(function(files) { - // console.log("Files returned by server:", files); - - // const zip = new JSZip(); - // const total = files.length; - // let completed = 0; - - // const fetchFiles = files.map(async file => { - // const response = await fetch(file.url); - // const blob = await response.blob(); - - // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - - // // Update progress - // completed++; - // const percent = Math.floor((completed / total) * 100); - // progressEl.value = percent; - // // progressBar.progress({ percent: percent }); - // textEl.textContent = `${completed} / ${total} files (${percent}%)`; - // }); - - // Promise.all(fetchFiles).then(() => { - // textEl.textContent = "Generating bundle"; - // progressEl.style.display="none"; - - // zip.generateAsync({ type: 'blob' }).then(blob => { - // const link = document.createElement('a'); - // link.href = URL.createObjectURL(blob); - // link.download = 'bulk_submissions.zip'; - // document.body.appendChild(link); - // link.click(); - // document.body.removeChild(link); - - // textEl.textContent = "Download ready!"; - // setTimeout(() => { - // statusBox.style.display = "none"; - // }, 3000); - // }); - // }); - // }) - // .fail(function(err) { - // console.error("Error downloading submissions:", err); - // textEl.textContent = "Error downloading!"; - // }); + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Support both jQuery jqXHR (.done/.fail) and native Promise (.then/.catch) + if (req && typeof req.done === "function") { + // jQuery-style + req.done(function() { + // jQuery passes (data, textStatus, jqXHR) + // forward only the first argument to our handler + const args = Array.from(arguments); + handleSuccess(args[0]); + }).fail(handleError); + } else if (req && typeof req.then === "function") { + // native Promise + req.then(handleSuccess).catch(handleError); + } else { + console.error("download_many_submissions returned non-promise/non-jqXHR:", req); + handleError(new Error("Invalid request return type")); + } }; self.submission_handling = function () { From 1ada62dff05f8b2a0b8e5a1f7321eac068f49002 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 19 Sep 2025 15:22:33 +0200 Subject: [PATCH 07/21] it should work --- src/apps/api/views/submissions.py | 31 ++++++++++++++----- .../detail/submission_manager.tag | 2 -- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 4be8895d9..b1ef247ee 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -19,7 +19,7 @@ from profiles.models import Organization, Membership from tasks.models import Task -from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer +from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer,SubmissionDetailSerializer from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard @@ -353,6 +353,8 @@ def re_run_many_submissions(self, request): @action(detail=False, methods=('POST',)) def download_many(self, request): + # logger.warning("Starting download Many.") + pks = request.data.get('pks') if not pks: return Response({"error": "`pks` field is required"}, status=400) @@ -361,17 +363,26 @@ def download_many(self, request): if not isinstance(pks, list): return Response({"error": "`pks` must be a list"}, status=400) + + # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", "data" - ) - # .only("id","owner", "data__data_file") + )#.only("id","owner", "data__data_file") + + # logger.warning("Submissions fetch from DB.") + + # .prefetch_related("phase__competition__collaborators") - if submissions.count() != len(pks): + + # .count() + if len(list(submissions)) != len(pks): + # if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: @@ -394,13 +405,19 @@ def download_many(self, request): ) files = [] + + # logger.warning("Submissions security passed") + + + for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" - - url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] + # url = sub.data.data_file.url + url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] + # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] files.append({"name": short_name, "url": url}) - + return Response(files) @action(detail=True, methods=('GET',)) diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 148980fdc..39bcacb7c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -715,7 +715,6 @@ let completed = 0; const failed = []; - // concurrency limit (tune this; 5-10 is usually a good start) const limit = 5; const queue = files.slice(); // clone const running = []; @@ -754,7 +753,6 @@ await Promise.race(running); } } - // wait for remaining ones await Promise.all(running); // Add failed.txt if necessary From 102ec8453f7012d6361579219a56b335838b6d7a Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 19 Sep 2025 15:46:49 +0200 Subject: [PATCH 08/21] cleaned comments --- src/apps/api/views/submissions.py | 10 ---------- src/static/js/ours/client.js | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index b1ef247ee..edea2f372 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -353,8 +353,6 @@ def re_run_many_submissions(self, request): @action(detail=False, methods=('POST',)) def download_many(self, request): - # logger.warning("Starting download Many.") - pks = request.data.get('pks') if not pks: return Response({"error": "`pks` field is required"}, status=400) @@ -372,12 +370,8 @@ def download_many(self, request): "data" )#.only("id","owner", "data__data_file") - # logger.warning("Submissions fetch from DB.") - - # .prefetch_related("phase__competition__collaborators") - # .count() if len(list(submissions)) != len(pks): # if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) @@ -406,10 +400,6 @@ def download_many(self, request): files = [] - # logger.warning("Submissions security passed") - - - for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index b3a8291c1..e4ffaa3fc 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -135,17 +135,6 @@ CODALAB.api = { ); }, - // download_many_submissions: function (pks) { - // return CODALAB.api.request('POST', `${URLS.API}submissions/download_many/`, { - // body: JSON.stringify({ pks: pks }), - // headers: { - // "Content-Type": "application/json" - // } - // }); - // const params = new URLSearchParams({ pks: JSON.stringify(pks) }); - // const url = `${URLS.API}submissions/download_many/?${params}`; - // return CODALAB.api.request('GET', url) - // }, /*--------------------------------------------------------------------- Leaderboards ---------------------------------------------------------------------*/ From 48a7f2823410a84fbb2a77f53f344cd9fd9cb5ab Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 23 Sep 2025 10:51:45 +0200 Subject: [PATCH 09/21] Flake8 fixes --- src/apps/api/views/submissions.py | 21 ++++++--------------- src/apps/competitions/tasks.py | 4 ---- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index edea2f372..15268e11e 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -15,11 +15,10 @@ from rest_framework.viewsets import ModelViewSet from rest_framework_csv import renderers from django.core.files.base import ContentFile -from django.http import StreamingHttpResponse from profiles.models import Organization, Membership from tasks.models import Task -from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer,SubmissionDetailSerializer +from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer, SubmissionDetailSerializer from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard @@ -349,8 +348,6 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - - @action(detail=False, methods=('POST',)) def download_many(self, request): pks = request.data.get('pks') @@ -360,24 +357,18 @@ def download_many(self, request): # pks is already parsed as a list if JSON was sent properly if not isinstance(pks, list): return Response({"error": "`pks` must be a list"}, status=400) - - # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", "data" - )#.only("id","owner", "data__data_file") - - # .prefetch_related("phase__competition__collaborators") + ) - if len(list(submissions)) != len(pks): - # if submissions.count() != len(pks): + if len(list(submissions)) != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) - - # NH : should should create a function for this ? + # Nicolas Homberg : should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -407,9 +398,9 @@ def download_many(self, request): url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] files.append({"name": short_name, "url": url}) - + return Response(files) - + @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 124711707..dc7242ff9 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -20,10 +20,6 @@ from django.utils.timezone import now from rest_framework.exceptions import ValidationError -from urllib.request import urlopen -from contextlib import closing -from urllib.error import ContentTooShortError - from celery_config import app from competitions.models import Submission, CompetitionCreationTaskStatus, SubmissionDetails, Competition, \ CompetitionDump, Phase From ee5a9eefdf97f7c75fc6f0318c6219d340c88f09 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Wed, 1 Oct 2025 16:41:35 +0200 Subject: [PATCH 10/21] added option to bulk download scores files and predictions files --- src/apps/api/views/submissions.py | 124 ++++++++++++---- src/static/js/ours/client.js | 14 ++ .../detail/submission_manager.tag | 133 +++++------------- 3 files changed, 148 insertions(+), 123 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index edea2f372..27289c631 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -186,6 +186,28 @@ def destroy(self, request, *args, **kwargs): self.perform_destroy(submission) return Response(status=status.HTTP_204_NO_CONTENT) + + + def check_submission_permissions(self,request,submissions): + # Check permissions + if not request.user.is_authenticated: + raise PermissionDenied("You must be logged in to download submissions") + # Allow admins + if request.user.is_superuser or request.user.is_staff: + allowed = True + else: + # Build one Q object for "owner OR organizer" + organiser_q = ( + Q(phase__competition__created_by=request.user) | + Q(phase__competition__collaborators=request.user) + ) + # Submissions that violate the rule + disallowed = submissions.exclude(Q(owner=request.user) | organiser_q) + allowed = not disallowed.exists() + if not allowed: + raise PermissionDenied( + "You do not have permission to download one or more of the requested submissions" + ) @action(detail=True, methods=('DELETE',)) def soft_delete(self, request, pk): @@ -350,7 +372,7 @@ def re_run_many_submissions(self, request): return Response({}) - +#todo : The 3 functions download many should be bundled inside a genereic with the function like "get_prediction_result" as a parameter instead of the same code 3 times. @action(detail=False, methods=('POST',)) def download_many(self, request): pks = request.data.get('pks') @@ -361,48 +383,23 @@ def download_many(self, request): if not isinstance(pks, list): return Response({"error": "`pks` must be a list"}, status=400) - - # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", "data" - )#.only("id","owner", "data__data_file") - - # .prefetch_related("phase__competition__collaborators") + ) if len(list(submissions)) != len(pks): - # if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) - - # NH : should should create a function for this ? - # Check permissions - if not request.user.is_authenticated: - raise PermissionDenied("You must be logged in to download submissions") - # Allow admins - if request.user.is_superuser or request.user.is_staff: - allowed = True - else: - # Build one Q object for "owner OR organizer" - organiser_q = ( - Q(phase__competition__created_by=request.user) | - Q(phase__competition__collaborators=request.user) - ) - # Submissions that violate the rule - disallowed = submissions.exclude(Q(owner=request.user) | organiser_q) - allowed = not disallowed.exists() - if not allowed: - raise PermissionDenied( - "You do not have permission to download one or more of the requested submissions" - ) + self.check_submission_permissions(request,submissions) files = [] for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] - short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" + short_name = f"sub_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" # url = sub.data.data_file.url url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] @@ -410,6 +407,75 @@ def download_many(self, request): return Response(files) + @action(detail=False, methods=('POST',)) + def download_many_prediction(self, request): + logger.warning("Download Many Preds.") + + pks = request.data.get('pks') + if not pks: + return Response({"error": "`pks` field is required"}, status=400) + + if not isinstance(pks, list): + return Response({"error": "`pks` must be a list"}, status=400) + + submissions = Submission.objects.filter(pk__in=pks).select_related( + "owner", + "phase", + "data" + ) + + if len(list(submissions)) != len(pks): + return Response({"error": "One or more submission IDs are invalid"}, status=404) + + self.check_submission_permissions(request,submissions) + + files = [] + serializer = SubmissionFilesSerializer(context=self.get_serializer_context()) + + for sub in submissions: + file_path = sub.data.data_file.name.split('/')[-1] + detailed_name = f"pred_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" + prediction_url = serializer.get_prediction_result(sub) + files.append({"name": detailed_name, "url": prediction_url}) + + return Response(files) + + @action(detail=False, methods=('POST',)) + def download_many_results(self, request): + logger.warning("Download Many Results.") + + pks = request.data.get('pks') + if not pks: + return Response({"error": "`pks` field is required"}, status=400) + + if not isinstance(pks, list): + return Response({"error": "`pks` must be a list"}, status=400) + + submissions = Submission.objects.filter(pk__in=pks).select_related( + "owner", + "phase", + "data" + ) + + if len(list(submissions)) != len(pks): + return Response({"error": "One or more submission IDs are invalid"}, status=404) + + self.check_submission_permissions(request,submissions) + + files = [] + serializer = SubmissionFilesSerializer(context=self.get_serializer_context()) + + + for sub in submissions: + file_path = sub.data.data_file.name.split('/')[-1] + complete_name = f"res_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" + result_url = serializer.get_scoring_result(sub) + #detailed results is already in the results zip file + # detailed_result_url = serializer.get_scoring_result(sub) + files.append({"name": complete_name, "url": result_url}) + + return Response(files) + @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index e4ffaa3fc..9f7fa93b3 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -134,6 +134,20 @@ CODALAB.api = { { pks: pks } // body is JSON by convention ); }, + download_many_submissions_prediction: function (pks) { + return CODALAB.api.request( + 'POST', + URLS.API + "submissions/download_many_prediction/", + { pks: pks } // body is JSON by convention + ); + }, + download_many_submissions_results: function (pks) { + return CODALAB.api.request( + 'POST', + URLS.API + "submissions/download_many_results/", + { pks: pks } // body is JSON by convention + ); + }, /*--------------------------------------------------------------------- Leaderboards diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 39bcacb7c..d4a407367 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -17,7 +17,10 @@ @@ -574,95 +577,9 @@ .modal('show') CODALAB.events.trigger('submission_clicked') } - - - // self.bulk_download = function () { - // const statusBox = document.getElementById('downloadStatus'); - // const progressEl = document.getElementById('downloadProgress'); - // const textEl = document.getElementById('progressText'); - - // statusBox.style.display = "flex"; - // // statusBox.style.display = "inline"; + self.bulk_download = function(files_type) { - // progressEl.style.display="flex"; - // progressEl.value = 0; - // textEl.textContent = "Preparing download..."; - - // console.log("Files returned by server:", files); - - - // CODALAB.api.download_many_submissions(self.checked_submissions) - // .done( async function(files) { - // // .done( function(files) { - // console.log("Files returned by server:", files); - - // const zip = new JSZip(); - // const total = files.length; - // let completed = 0; - // const failed = []; - - // const fetchFiles = files.map(async file => { - // try { - // const response = await fetch(file.url); - - // if (!response.ok) { - // throw new Error(`HTTP ${response.status}`); - // } - - // const blob = await response.blob(); - - // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - // } catch (err) { - // console.error(`Failed to fetch ${file.name}:`, err); - // failed.push(file.name); - // } finally { - // // Update progress regardless of success/failure - // completed++; - // const percent = Math.floor((completed / total) * 100); - // progressEl.value = percent; - // textEl.textContent = `${completed} / ${total} files (${percent}%)`; - // } - // }); - - // Promise.allSettled(fetchFiles).then(() => { - // // If some files failed, include them as failed.txt inside the zip - // if (failed.length > 0) { - // const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; - // zip.file("failed.txt", failedContent); - // } - - - // textEl.textContent = "Generating bundle"; - // progressEl.style.display = "none"; - - // zip.generateAsync({ type: "blob" }).then(blob => { - // const link = document.createElement("a"); - // link.href = URL.createObjectURL(blob); - // link.download = "bulk_submissions.zip"; - // document.body.appendChild(link); - // link.click(); - // document.body.removeChild(link); - - // if (failed.length > 0) { - // textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; - // } else { - // textEl.textContent = "Download ready!"; - // } - - // setTimeout(() => { - // statusBox.style.display = "none"; - // }, 5000); - // }); - // }); - // }) - // .fail(function(err) { - // console.error("Error downloading submissions:", err); - // textEl.textContent = "Error downloading!"; - // }); - // }; - - self.bulk_download = function () { const statusBox = document.getElementById('downloadStatus'); const progressEl = document.getElementById('downloadProgress'); const textEl = document.getElementById('progressText'); @@ -672,8 +589,30 @@ progressEl.value = 0; textEl.textContent = "Preparing download..."; - // Kick the API request - const req = CODALAB.api.download_many_submissions(self.checked_submissions); + let req ; + switch (files_type) { + case "submissions": + // self.bulk_download("download") + req = CODALAB.api.download_many_submissions(self.checked_submissions); + + break; + case "results": + console.log("download_results") + req = CODALAB.api.download_many_submissions_results(self.checked_submissions); + + // self.bulk_download_results("download_results") + break; + case "predictions": + console.log("download_prediction") + req = CODALAB.api.download_many_submissions_prediction(self.checked_submissions); + // self.bulk_download_prediction("download_prediction") + break; + default: + console.error("Default switch case in bulk function") + return; + } + + // const req = CODALAB.api.download_many_submissions(self.checked_submissions); // Common error handler const handleError = (err) => { @@ -767,7 +706,7 @@ const blob = await zip.generateAsync({ type: "blob" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); - link.download = "bulk_submissions.zip"; + link.download = "bulk_" +files_type +".zip"; document.body.appendChild(link); link.click(); document.body.removeChild(link); @@ -807,12 +746,18 @@ var submission_operation = self.refs.submission_handling_operation.value switch (submission_operation) { case "delete": - // console.log("delete") self.delete_selected_submissions() break; case "download": - // console.log("download") - self.bulk_download() + self.bulk_download("submissions") + break; + case "download_results": + console.log("download_results") + self.bulk_download("results") + break; + case "download_prediction": + console.log("download_prediction") + self.bulk_download("predictions") break; case "rerun": // console.log("rerun") From 20ff485b083c5272ec7ca0e75b151c25cc1c6d18 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Thu, 2 Oct 2025 11:59:36 +0200 Subject: [PATCH 11/21] cleaned code --- src/apps/api/views/submissions.py | 4 +--- .../competitions/detail/submission_manager.tag | 17 +---------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 27289c631..71fca6782 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -372,7 +372,7 @@ def re_run_many_submissions(self, request): return Response({}) -#todo : The 3 functions download many should be bundled inside a genereic with the function like "get_prediction_result" as a parameter instead of the same code 3 times. + #Todo : The 3 functions download many should be bundled inside a genereic with the function like "get_prediction_result" as a parameter instead of the same code 3 times. @action(detail=False, methods=('POST',)) def download_many(self, request): pks = request.data.get('pks') @@ -409,7 +409,6 @@ def download_many(self, request): @action(detail=False, methods=('POST',)) def download_many_prediction(self, request): - logger.warning("Download Many Preds.") pks = request.data.get('pks') if not pks: @@ -442,7 +441,6 @@ def download_many_prediction(self, request): @action(detail=False, methods=('POST',)) def download_many_results(self, request): - logger.warning("Download Many Results.") pks = request.data.get('pks') if not pks: diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index d4a407367..5152744eb 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -592,28 +592,19 @@ let req ; switch (files_type) { case "submissions": - // self.bulk_download("download") req = CODALAB.api.download_many_submissions(self.checked_submissions); - break; case "results": - console.log("download_results") req = CODALAB.api.download_many_submissions_results(self.checked_submissions); - - // self.bulk_download_results("download_results") break; case "predictions": - console.log("download_prediction") req = CODALAB.api.download_many_submissions_prediction(self.checked_submissions); - // self.bulk_download_prediction("download_prediction") break; default: console.error("Default switch case in bulk function") return; } - // const req = CODALAB.api.download_many_submissions(self.checked_submissions); - // Common error handler const handleError = (err) => { console.error("Error downloading submissions:", err); @@ -621,18 +612,15 @@ setTimeout(() => { statusBox.style.display = "none"; }, 5000); }; - // Success handler (async because we await inside) const handleSuccess = async (resp) => { // Normalize response -> files array let files = resp; if (resp && typeof resp === 'object' && !Array.isArray(resp)) { - // common shapes: { files: [...] } or { data: [...] } or direct array if (Array.isArray(resp.files)) files = resp.files; else if (Array.isArray(resp.data)) files = resp.data; else if (Array.isArray(resp.results)) files = resp.results; else if (Array.isArray(resp)) files = resp; else { - // if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0]; else { console.warn("Unexpected response shape from download_many_submissions:", resp); @@ -647,7 +635,7 @@ return; } - console.log("Files returned by server:", files); + // console.log("Files returned by server:", files); const zip = new JSZip(); const total = files.length; @@ -752,15 +740,12 @@ self.bulk_download("submissions") break; case "download_results": - console.log("download_results") self.bulk_download("results") break; case "download_prediction": - console.log("download_prediction") self.bulk_download("predictions") break; case "rerun": - // console.log("rerun") self.rerun_selected_submissions() break default: From 0886f7c12f6c269108a0072e987a0a78e62cccd5 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 3 Oct 2025 11:04:47 +0200 Subject: [PATCH 12/21] exclude not finished submission inside download pred and scores --- src/apps/api/views/submissions.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 71fca6782..5b805c0bc 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -432,11 +432,12 @@ def download_many_prediction(self, request): serializer = SubmissionFilesSerializer(context=self.get_serializer_context()) for sub in submissions: + if sub.status not in [ Submission.FINISHED]: #Submission.FAILED, Submission.CANCELLED + continue file_path = sub.data.data_file.name.split('/')[-1] detailed_name = f"pred_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" prediction_url = serializer.get_prediction_result(sub) files.append({"name": detailed_name, "url": prediction_url}) - return Response(files) @action(detail=False, methods=('POST',)) @@ -465,10 +466,12 @@ def download_many_results(self, request): for sub in submissions: + if sub.status not in [ Submission.FINISHED]: #Submission.FAILED, Submission.CANCELLED + continue file_path = sub.data.data_file.name.split('/')[-1] complete_name = f"res_{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" result_url = serializer.get_scoring_result(sub) - #detailed results is already in the results zip file + #detailed results is already in the results zip file but For very large detailed results it could be helpfull to remove it. # detailed_result_url = serializer.get_scoring_result(sub) files.append({"name": complete_name, "url": result_url}) From 7610a22d85fc243f13fe3a36c35780812ec18c63 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Thu, 4 Sep 2025 23:29:42 +0200 Subject: [PATCH 13/21] download client side and bulk download --- src/apps/api/views/submissions.py | 73 +++++++++++++++-- src/apps/competitions/tasks.py | 82 +++++++++---------- src/static/js/ours/client.js | 55 ++++++++----- .../detail/submission_manager.tag | 6 +- 4 files changed, 142 insertions(+), 74 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 1b89ec6ae..a6f1c3e83 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -382,6 +382,7 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) + @action(detail=False, methods=['get']) def download_many(self, request): """ @@ -396,12 +397,15 @@ def download_many(self, request): # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", - "phase__competition", - "phase__competition__created_by", - ).prefetch_related("phase__competition__collaborators") + "phase", + "data", + ) + # .only("id","owner", "data__data_file") + # .prefetch_related("phase__competition__collaborators") if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -422,12 +426,63 @@ def download_many(self, request): "You do not have permission to download one or more of the requested submissions" ) - # Download - from competitions.tasks import stream_batch_download - in_memory_zip = stream_batch_download(pks) - response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') - response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' - return response + files = [] + for sub in submissions: + file_path = sub.data.data_file.name.split('/')[-1] + short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" + + url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] + files.append({"name": short_name, "url": url}) + + return Response(files) + + + # @action(detail=False, methods=['get']) + # def download_many(self, request): + # """ + # Download a ZIP containing several submissions. + # """ + # pks = request.query_params.get('pks') + # if pks: + # pks = json.loads(pks) # Convert JSON string to list + # else: + # return Response({"error": "`pks` query parameter is required"}, status=400) + + # # Get submissions + # submissions = Submission.objects.filter(pk__in=pks).select_related( + # "owner", + # "phase__competition", + # "phase__competition__created_by", + # ).prefetch_related("phase__competition__collaborators") + # if submissions.count() != len(pks): + # return Response({"error": "One or more submission IDs are invalid"}, status=404) + + # # Check permissions + # if not request.user.is_authenticated: + # raise PermissionDenied("You must be logged in to download submissions") + # # Allow admins + # if request.user.is_superuser or request.user.is_staff: + # allowed = True + # else: + # # Build one Q object for "owner OR organizer" + # organiser_q = ( + # Q(phase__competition__created_by=request.user) | + # Q(phase__competition__collaborators=request.user) + # ) + # # Submissions that violate the rule + # disallowed = submissions.exclude(Q(owner=request.user) | organiser_q) + # allowed = not disallowed.exists() + # if not allowed: + # raise PermissionDenied( + # "You do not have permission to download one or more of the requested submissions" + # ) + + # # Download + # from competitions.tasks import stream_batch_download + # in_memory_zip = stream_batch_download(pks) + # response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') + # response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' + # return response @action(detail=True, methods=('GET',)) def get_details(self, request, pk): diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index f3f577139..96c2776ac 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -280,47 +280,47 @@ def send_child_id(submission, child_id): }) -def retrieve_data(url, data=None): - with closing(urlopen(url, data)) as fp: - headers = fp.info() - - bs = 1024 * 8 - size = -1 - read = 0 - if "content-length" in headers: - size = int(headers["Content-Length"]) - - while True: - block = fp.read(bs) - if not block: - break - read += len(block) - yield(block) - - if size >= 0 and read < size: - raise ContentTooShortError( - "retrieval incomplete: got only %i out of %i bytes" - % (read, size)) - - -def zip_generator(submission_pks): - in_memory_zip = BytesIO() - with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: - for submission_id in submission_pks: - submission = Submission.objects.get(id=submission_id) - short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] - url = make_url_sassy(path=submission.data.data_file.name) - for block in retrieve_data(url): - zip_file.writestr(short_name, block) - - in_memory_zip.seek(0) - - return in_memory_zip - - -@app.task(queue='site-worker', soft_time_limit=60 * 60) -def stream_batch_download(submission_pks): - return zip_generator(submission_pks) +# def retrieve_data(url, data=None): +# with closing(urlopen(url, data)) as fp: +# headers = fp.info() + +# bs = 1024 * 8 +# size = -1 +# read = 0 +# if "content-length" in headers: +# size = int(headers["Content-Length"]) + +# while True: +# block = fp.read(bs) +# if not block: +# break +# read += len(block) +# yield(block) + +# if size >= 0 and read < size: +# raise ContentTooShortError( +# "retrieval incomplete: got only %i out of %i bytes" +# % (read, size)) + + +# def zip_generator(submission_pks): +# in_memory_zip = BytesIO() +# with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: +# for submission_id in submission_pks: +# submission = Submission.objects.get(id=submission_id) +# short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] +# url = make_url_sassy(path=submission.data.data_file.name) +# for block in retrieve_data(url): +# zip_file.writestr(short_name, block) + +# in_memory_zip.seek(0) + +# return in_memory_zip + + +# @app.task(queue='site-worker', soft_time_limit=60 * 60) +# def stream_batch_download(submission_pks): +# return zip_generator(submission_pks) @app.task(queue='site-worker', soft_time_limit=60) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 642a3ccfd..c1a4bae5c 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -128,29 +128,42 @@ CODALAB.api = { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, download_many_submissions: function (pks) { - console.log('Request bulk'); const params = new URLSearchParams({ pks: JSON.stringify(pks) }); const url = `${URLS.API}submissions/download_many/?${params}`; - return fetch(url, { - method: 'GET', - headers: { - 'Content-Type': 'application/json' - } - }).then(response => { - if (!response.ok) { - throw new Error('Network response was not ok ' + response.statusText); - } - return response.blob(); - }).then(blob => { - const link = document.createElement('a'); - link.href = window.URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }).catch(error => { - console.error('Error downloading submissions:', error); - }); + // return CODALAB.api.request('GET', url) + CODALAB.api.request('GET', url) + .done(function(files) { + // files is already parsed JSON array + console.log("Files returned by server:", files); + + // Iterate files + files.forEach(file => { + console.log(file.name, file.url); + }); + // return files + + const zip = new JSZip(); + const fetchFiles = files.map(async file => { + const response = await fetch(file.url); + const blob = await response.blob(); + zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + }); + + Promise.all(fetchFiles).then(() => { + zip.generateAsync({ type: 'blob' }).then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }); + }); + + }) + .fail(function(err) { + console.error("Error downloading submissions:", err); + }); }, /*--------------------------------------------------------------------- diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 74b746cc6..03e0c1c9e 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -571,9 +571,9 @@ self.bulk_download = function () { CODALAB.api.download_many_submissions(self.checked_submissions) - .catch(function (error) { - console.error('Error:', error); - }); + #.catch(function (error) { + # console.error('Error:', error); + #}); } From 2726a2402144f42fd62d097e8085bca9e5e4556c Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Mon, 8 Sep 2025 16:44:57 +0200 Subject: [PATCH 14/21] working bulk download --- src/static/js/ours/client.js | 58 +++++------ .../detail/submission_manager.tag | 97 +++++++++++++++---- src/templates/base.html | 1 + 3 files changed, 107 insertions(+), 49 deletions(-) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index c1a4bae5c..a2b05a35c 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -131,39 +131,39 @@ CODALAB.api = { const params = new URLSearchParams({ pks: JSON.stringify(pks) }); const url = `${URLS.API}submissions/download_many/?${params}`; // return CODALAB.api.request('GET', url) - CODALAB.api.request('GET', url) - .done(function(files) { - // files is already parsed JSON array - console.log("Files returned by server:", files); + return CODALAB.api.request('GET', url) + // .done(function(files) { + // // files is already parsed JSON array + // console.log("Files returned by server:", files); - // Iterate files - files.forEach(file => { - console.log(file.name, file.url); - }); - // return files + // // Iterate files + // files.forEach(file => { + // console.log(file.name, file.url); + // }); + // // return files - const zip = new JSZip(); - const fetchFiles = files.map(async file => { - const response = await fetch(file.url); - const blob = await response.blob(); - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - }); + // const zip = new JSZip(); + // const fetchFiles = files.map(async file => { + // const response = await fetch(file.url); + // const blob = await response.blob(); + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + // }); - Promise.all(fetchFiles).then(() => { - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); - link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }); - }); + // Promise.all(fetchFiles).then(() => { + // zip.generateAsync({ type: 'blob' }).then(blob => { + // const link = document.createElement('a'); + // link.href = URL.createObjectURL(blob); + // link.download = 'bulk_submissions.zip'; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + // }); + // }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // }); }, /*--------------------------------------------------------------------- diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 03e0c1c9e..79d894bc3 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -1,29 +1,36 @@ -
-
+
+
+
Rerun all submissions per phase
Select a phase
-
- { phase.name } +
+ { phase.name }
-
- +
+ + Download as CSV - + - + - - + +
+
@@ -568,15 +575,65 @@ CODALAB.events.trigger('submission_clicked') } - self.bulk_download = function () { - CODALAB.api.download_many_submissions(self.checked_submissions) - #.catch(function (error) { - # console.error('Error:', error); - #}); - } + const statusBox = document.getElementById('downloadStatus'); + const progressEl = document.getElementById('downloadProgress'); + const textEl = document.getElementById('progressText'); + statusBox.style.display = "flex"; + // statusBox.style.display = "inline"; + + progressEl.style.display="flex"; + progressEl.value = 0; + textEl.textContent = "Preparing download..."; + + CODALAB.api.download_many_submissions(self.checked_submissions) + .done(function(files) { + console.log("Files returned by server:", files); + + const zip = new JSZip(); + const total = files.length; + let completed = 0; + + const fetchFiles = files.map(async file => { + const response = await fetch(file.url); + const blob = await response.blob(); + + zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + + // Update progress + completed++; + const percent = Math.round((completed / total) * 100); + progressEl.value = percent; + // progressBar.progress({ percent: percent }); + textEl.textContent = `${completed} / ${total} files (${percent}%)`; + }); + + Promise.all(fetchFiles).then(() => { + textEl.textContent = "Generating bundle"; + progressEl.style.display="none"; + + zip.generateAsync({ type: 'blob' }).then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + textEl.textContent = "Download ready!"; + setTimeout(() => { + statusBox.style.display = "none"; + }, 3000); + }); + }); + }) + .fail(function(err) { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + }); + }; self.submission_handling = function () { // console.log(self.checked_submissions) diff --git a/src/templates/base.html b/src/templates/base.html index 70c39b486..cfceb0672 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -228,6 +228,7 @@

CodaBench

+ From 61d5f1194b0c3dad4b1ede5f3f41abde599b4847 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Mon, 8 Sep 2025 17:19:50 +0200 Subject: [PATCH 15/21] cleaned --- src/apps/api/views/submissions.py | 50 +------------------------------ src/apps/competitions/tasks.py | 43 -------------------------- src/static/js/ours/client.js | 33 -------------------- 3 files changed, 1 insertion(+), 125 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index a6f1c3e83..a2386bcc9 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -398,7 +398,7 @@ def download_many(self, request): submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", - "data", + "data" ) # .only("id","owner", "data__data_file") # .prefetch_related("phase__competition__collaborators") @@ -436,54 +436,6 @@ def download_many(self, request): return Response(files) - - # @action(detail=False, methods=['get']) - # def download_many(self, request): - # """ - # Download a ZIP containing several submissions. - # """ - # pks = request.query_params.get('pks') - # if pks: - # pks = json.loads(pks) # Convert JSON string to list - # else: - # return Response({"error": "`pks` query parameter is required"}, status=400) - - # # Get submissions - # submissions = Submission.objects.filter(pk__in=pks).select_related( - # "owner", - # "phase__competition", - # "phase__competition__created_by", - # ).prefetch_related("phase__competition__collaborators") - # if submissions.count() != len(pks): - # return Response({"error": "One or more submission IDs are invalid"}, status=404) - - # # Check permissions - # if not request.user.is_authenticated: - # raise PermissionDenied("You must be logged in to download submissions") - # # Allow admins - # if request.user.is_superuser or request.user.is_staff: - # allowed = True - # else: - # # Build one Q object for "owner OR organizer" - # organiser_q = ( - # Q(phase__competition__created_by=request.user) | - # Q(phase__competition__collaborators=request.user) - # ) - # # Submissions that violate the rule - # disallowed = submissions.exclude(Q(owner=request.user) | organiser_q) - # allowed = not disallowed.exists() - # if not allowed: - # raise PermissionDenied( - # "You do not have permission to download one or more of the requested submissions" - # ) - - # # Download - # from competitions.tasks import stream_batch_download - # in_memory_zip = stream_batch_download(pks) - # response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') - # response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' - # return response - @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 96c2776ac..631aa0815 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -280,49 +280,6 @@ def send_child_id(submission, child_id): }) -# def retrieve_data(url, data=None): -# with closing(urlopen(url, data)) as fp: -# headers = fp.info() - -# bs = 1024 * 8 -# size = -1 -# read = 0 -# if "content-length" in headers: -# size = int(headers["Content-Length"]) - -# while True: -# block = fp.read(bs) -# if not block: -# break -# read += len(block) -# yield(block) - -# if size >= 0 and read < size: -# raise ContentTooShortError( -# "retrieval incomplete: got only %i out of %i bytes" -# % (read, size)) - - -# def zip_generator(submission_pks): -# in_memory_zip = BytesIO() -# with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: -# for submission_id in submission_pks: -# submission = Submission.objects.get(id=submission_id) -# short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] -# url = make_url_sassy(path=submission.data.data_file.name) -# for block in retrieve_data(url): -# zip_file.writestr(short_name, block) - -# in_memory_zip.seek(0) - -# return in_memory_zip - - -# @app.task(queue='site-worker', soft_time_limit=60 * 60) -# def stream_batch_download(submission_pks): -# return zip_generator(submission_pks) - - @app.task(queue='site-worker', soft_time_limit=60) def _run_submission(submission_pk, task_pks=None, is_scoring=False): """This function is wrapped so that when we run tests we can run this function not diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index a2b05a35c..a48270914 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -130,40 +130,7 @@ CODALAB.api = { download_many_submissions: function (pks) { const params = new URLSearchParams({ pks: JSON.stringify(pks) }); const url = `${URLS.API}submissions/download_many/?${params}`; - // return CODALAB.api.request('GET', url) return CODALAB.api.request('GET', url) - // .done(function(files) { - // // files is already parsed JSON array - // console.log("Files returned by server:", files); - - // // Iterate files - // files.forEach(file => { - // console.log(file.name, file.url); - // }); - // // return files - - // const zip = new JSZip(); - // const fetchFiles = files.map(async file => { - // const response = await fetch(file.url); - // const blob = await response.blob(); - // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - // }); - - // Promise.all(fetchFiles).then(() => { - // zip.generateAsync({ type: 'blob' }).then(blob => { - // const link = document.createElement('a'); - // link.href = URL.createObjectURL(blob); - // link.download = 'bulk_submissions.zip'; - // document.body.appendChild(link); - // link.click(); - // document.body.removeChild(link); - // }); - // }); - - // }) - // .fail(function(err) { - // console.error("Error downloading submissions:", err); - // }); }, /*--------------------------------------------------------------------- From 279faf54eed47d09551fb188d19b89d0a5aa4fd7 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Wed, 17 Sep 2025 15:29:58 +0200 Subject: [PATCH 16/21] gestion of faulty submission --- src/apps/api/views/submissions.py | 5 +- .../detail/submission_manager.tag | 150 +++++++++++++----- 2 files changed, 108 insertions(+), 47 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index a2386bcc9..4465f7486 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -382,7 +382,6 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - @action(detail=False, methods=['get']) def download_many(self, request): """ @@ -405,7 +404,7 @@ def download_many(self, request): if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) - # NH : should should create a function for this ? + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -426,7 +425,7 @@ def download_many(self, request): "You do not have permission to download one or more of the requested submissions" ) - files = [] + files = [] for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 79d894bc3..80f0cc5ef 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -574,7 +574,8 @@ .modal('show') CODALAB.events.trigger('submission_clicked') } - + + self.bulk_download = function () { const statusBox = document.getElementById('downloadStatus'); const progressEl = document.getElementById('downloadProgress'); @@ -589,50 +590,111 @@ textEl.textContent = "Preparing download..."; CODALAB.api.download_many_submissions(self.checked_submissions) - .done(function(files) { - console.log("Files returned by server:", files); - - const zip = new JSZip(); - const total = files.length; - let completed = 0; - - const fetchFiles = files.map(async file => { - const response = await fetch(file.url); - const blob = await response.blob(); - - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - - // Update progress - completed++; - const percent = Math.round((completed / total) * 100); - progressEl.value = percent; - // progressBar.progress({ percent: percent }); - textEl.textContent = `${completed} / ${total} files (${percent}%)`; - }); - - Promise.all(fetchFiles).then(() => { - textEl.textContent = "Generating bundle"; - progressEl.style.display="none"; - - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); - link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - - textEl.textContent = "Download ready!"; - setTimeout(() => { - statusBox.style.display = "none"; - }, 3000); - }); - }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - textEl.textContent = "Error downloading!"; + .done(function(files) { + console.log("Files returned by server:", files); + + const zip = new JSZip(); + const total = files.length; + let completed = 0; + const failed = []; + + const fetchFiles = files.map(async file => { + try { + const response = await fetch(file.url); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const blob = await response.blob(); + + zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + } catch (err) { + console.error(`Failed to fetch ${file.name}:`, err); + failed.push(file.name); + } finally { + // Update progress regardless of success/failure + completed++; + const percent = Math.floor((completed / total) * 100); + progressEl.value = percent; + textEl.textContent = `${completed} / ${total} files (${percent}%)`; + } }); + + Promise.allSettled(fetchFiles).then(() => { + textEl.textContent = "Generating bundle"; + progressEl.style.display = "none"; + + zip.generateAsync({ type: 'blob' }).then(blob => { + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'bulk_submissions.zip'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + if (failed.length > 0) { + textEl.textContent = `Download complete, but ${failed.length} failed: ${failed.join(', ')}`; + } else { + textEl.textContent = "Download ready!"; + } + + setTimeout(() => { + statusBox.style.display = "none"; + }, 5000); + }); + }); + }) + .fail(function(err) { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + }); + + // CODALAB.api.download_many_submissions(self.checked_submissions) + // .done(function(files) { + // console.log("Files returned by server:", files); + + // const zip = new JSZip(); + // const total = files.length; + // let completed = 0; + + // const fetchFiles = files.map(async file => { + // const response = await fetch(file.url); + // const blob = await response.blob(); + + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + + // // Update progress + // completed++; + // const percent = Math.floor((completed / total) * 100); + // progressEl.value = percent; + // // progressBar.progress({ percent: percent }); + // textEl.textContent = `${completed} / ${total} files (${percent}%)`; + // }); + + // Promise.all(fetchFiles).then(() => { + // textEl.textContent = "Generating bundle"; + // progressEl.style.display="none"; + + // zip.generateAsync({ type: 'blob' }).then(blob => { + // const link = document.createElement('a'); + // link.href = URL.createObjectURL(blob); + // link.download = 'bulk_submissions.zip'; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + + // textEl.textContent = "Download ready!"; + // setTimeout(() => { + // statusBox.style.display = "none"; + // }, 3000); + // }); + // }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // textEl.textContent = "Error downloading!"; + // }); }; self.submission_handling = function () { From 78f9de956cd02d618c9e5a93c3e4b3ea7a146b8b Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Thu, 18 Sep 2025 15:02:46 +0200 Subject: [PATCH 17/21] changed to post instead of get and added failed.txt to bundle --- src/apps/api/views/submissions.py | 29 ++++++++++++------- src/static/js/ours/client.js | 21 +++++++++++--- .../detail/submission_manager.tag | 14 ++++++--- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 4465f7486..6203d9df3 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -382,17 +382,26 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - @action(detail=False, methods=['get']) + # @action(detail=False, methods=['get']) + # def download_many(self, request): + # """ + # Download a ZIP containing several submissions. + # """ + # pks = request.query_params.get('pks') + # if pks: + # pks = json.loads(pks) # Convert JSON string to list + # else: + # return Response({"error": "`pks` query parameter is required"}, status=400) + @action(detail=False, methods=['POST']) def download_many(self, request): - """ - Download a ZIP containing several submissions. - """ - pks = request.query_params.get('pks') - if pks: - pks = json.loads(pks) # Convert JSON string to list - else: - return Response({"error": "`pks` query parameter is required"}, status=400) - + pks = request.data.get('pks') + if not pks: + return Response({"error": "`pks` field is required"}, status=400) + + # pks is already parsed as a list if JSON was sent properly + if not isinstance(pks, list): + return Response({"error": "`pks` must be a list"}, status=400) + # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index a48270914..9fbd37251 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -128,11 +128,24 @@ CODALAB.api = { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, download_many_submissions: function (pks) { - const params = new URLSearchParams({ pks: JSON.stringify(pks) }); - const url = `${URLS.API}submissions/download_many/?${params}`; - return CODALAB.api.request('GET', url) + return CODALAB.api.request( + 'POST', + URLS.API + "submissions/download_many/", + { pks: pks } // body is JSON by convention + ); }, - + + // download_many_submissions: function (pks) { + // return CODALAB.api.request('POST', `${URLS.API}submissions/download_many/`, { + // body: JSON.stringify({ pks: pks }), + // headers: { + // "Content-Type": "application/json" + // } + // }); + // const params = new URLSearchParams({ pks: JSON.stringify(pks) }); + // const url = `${URLS.API}submissions/download_many/?${params}`; + // return CODALAB.api.request('GET', url) + // }, /*--------------------------------------------------------------------- Leaderboards ---------------------------------------------------------------------*/ diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 80f0cc5ef..96ba0151a 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -622,19 +622,25 @@ }); Promise.allSettled(fetchFiles).then(() => { + // If some files failed, include them as failed.txt inside the zip + if (failed.length > 0) { + const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + zip.file("failed.txt", failedContent); + } + textEl.textContent = "Generating bundle"; progressEl.style.display = "none"; - zip.generateAsync({ type: 'blob' }).then(blob => { - const link = document.createElement('a'); + zip.generateAsync({ type: "blob" }).then(blob => { + const link = document.createElement("a"); link.href = URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; + link.download = "bulk_submissions.zip"; document.body.appendChild(link); link.click(); document.body.removeChild(link); if (failed.length > 0) { - textEl.textContent = `Download complete, but ${failed.length} failed: ${failed.join(', ')}`; + textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; } else { textEl.textContent = "Download ready!"; } From 86e27bee8ecea0222cec9f3c8b846d71d5921145 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 19 Sep 2025 12:01:47 +0200 Subject: [PATCH 18/21] added limit on number of file to download simultaneously --- src/apps/api/views/submissions.py | 14 +- .../detail/submission_manager.tag | 278 ++++++++++++------ 2 files changed, 191 insertions(+), 101 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 6203d9df3..e20b6f8b5 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -382,17 +382,9 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - # @action(detail=False, methods=['get']) - # def download_many(self, request): - # """ - # Download a ZIP containing several submissions. - # """ - # pks = request.query_params.get('pks') - # if pks: - # pks = json.loads(pks) # Convert JSON string to list - # else: - # return Response({"error": "`pks` query parameter is required"}, status=400) - @action(detail=False, methods=['POST']) + + + @action(detail=False, methods=('POST',)) def download_many(self, request): pks = request.data.get('pks') if not pks: diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 96ba0151a..148980fdc 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -575,22 +575,139 @@ CODALAB.events.trigger('submission_clicked') } - + + // self.bulk_download = function () { + // const statusBox = document.getElementById('downloadStatus'); + // const progressEl = document.getElementById('downloadProgress'); + // const textEl = document.getElementById('progressText'); + + // statusBox.style.display = "flex"; + // // statusBox.style.display = "inline"; + + + // progressEl.style.display="flex"; + // progressEl.value = 0; + // textEl.textContent = "Preparing download..."; + + // console.log("Files returned by server:", files); + + + // CODALAB.api.download_many_submissions(self.checked_submissions) + // .done( async function(files) { + // // .done( function(files) { + // console.log("Files returned by server:", files); + + // const zip = new JSZip(); + // const total = files.length; + // let completed = 0; + // const failed = []; + + // const fetchFiles = files.map(async file => { + // try { + // const response = await fetch(file.url); + + // if (!response.ok) { + // throw new Error(`HTTP ${response.status}`); + // } + + // const blob = await response.blob(); + + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + // } catch (err) { + // console.error(`Failed to fetch ${file.name}:`, err); + // failed.push(file.name); + // } finally { + // // Update progress regardless of success/failure + // completed++; + // const percent = Math.floor((completed / total) * 100); + // progressEl.value = percent; + // textEl.textContent = `${completed} / ${total} files (${percent}%)`; + // } + // }); + + // Promise.allSettled(fetchFiles).then(() => { + // // If some files failed, include them as failed.txt inside the zip + // if (failed.length > 0) { + // const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + // zip.file("failed.txt", failedContent); + // } + + + // textEl.textContent = "Generating bundle"; + // progressEl.style.display = "none"; + + // zip.generateAsync({ type: "blob" }).then(blob => { + // const link = document.createElement("a"); + // link.href = URL.createObjectURL(blob); + // link.download = "bulk_submissions.zip"; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + + // if (failed.length > 0) { + // textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + // } else { + // textEl.textContent = "Download ready!"; + // } + + // setTimeout(() => { + // statusBox.style.display = "none"; + // }, 5000); + // }); + // }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // textEl.textContent = "Error downloading!"; + // }); + // }; + self.bulk_download = function () { const statusBox = document.getElementById('downloadStatus'); const progressEl = document.getElementById('downloadProgress'); const textEl = document.getElementById('progressText'); statusBox.style.display = "flex"; - // statusBox.style.display = "inline"; - - - progressEl.style.display="flex"; + progressEl.style.display = "flex"; progressEl.value = 0; textEl.textContent = "Preparing download..."; - CODALAB.api.download_many_submissions(self.checked_submissions) - .done(function(files) { + // Kick the API request + const req = CODALAB.api.download_many_submissions(self.checked_submissions); + + // Common error handler + const handleError = (err) => { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Success handler (async because we await inside) + const handleSuccess = async (resp) => { + // Normalize response -> files array + let files = resp; + if (resp && typeof resp === 'object' && !Array.isArray(resp)) { + // common shapes: { files: [...] } or { data: [...] } or direct array + if (Array.isArray(resp.files)) files = resp.files; + else if (Array.isArray(resp.data)) files = resp.data; + else if (Array.isArray(resp.results)) files = resp.results; + else if (Array.isArray(resp)) files = resp; + else { + // if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg + if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0]; + else { + console.warn("Unexpected response shape from download_many_submissions:", resp); + files = []; + } + } + } + + if (!Array.isArray(files) || files.length === 0) { + textEl.textContent = "No files to download"; + setTimeout(() => { statusBox.style.display = "none"; }, 3000); + return; + } + console.log("Files returned by server:", files); const zip = new JSZip(); @@ -598,109 +715,90 @@ let completed = 0; const failed = []; - const fetchFiles = files.map(async file => { + // concurrency limit (tune this; 5-10 is usually a good start) + const limit = 5; + const queue = files.slice(); // clone + const running = []; + + // worker to fetch one file + async function worker(file) { try { const response = await fetch(file.url); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); - } - + if (!response.ok) throw new Error(`HTTP ${response.status}`); const blob = await response.blob(); - - zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + zip.file(file.name.replace(/[:/\\]/g, "_"), blob); } catch (err) { console.error(`Failed to fetch ${file.name}:`, err); - failed.push(file.name); + failed.push(`${file.name} (${err.message || 'fetch error'})`); } finally { - // Update progress regardless of success/failure completed++; const percent = Math.floor((completed / total) * 100); progressEl.value = percent; textEl.textContent = `${completed} / ${total} files (${percent}%)`; } - }); + } + + // run queue with limited concurrency + while (queue.length > 0) { + while (running.length < limit && queue.length > 0) { + const file = queue.shift(); + const p = worker(file).then(() => { + // remove finished promise from running + const idx = running.indexOf(p); + if (idx !== -1) running.splice(idx, 1); + }); + running.push(p); + } + // Wait for at least one running promise to finish + if (running.length > 0) { + await Promise.race(running); + } + } + // wait for remaining ones + await Promise.all(running); - Promise.allSettled(fetchFiles).then(() => { - // If some files failed, include them as failed.txt inside the zip + // Add failed.txt if necessary if (failed.length > 0) { - const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; - zip.file("failed.txt", failedContent); + const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + zip.file("failed.txt", failedContent); } textEl.textContent = "Generating bundle"; progressEl.style.display = "none"; - zip.generateAsync({ type: "blob" }).then(blob => { - const link = document.createElement("a"); - link.href = URL.createObjectURL(blob); - link.download = "bulk_submissions.zip"; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); + const blob = await zip.generateAsync({ type: "blob" }); + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = "bulk_submissions.zip"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); - if (failed.length > 0) { - textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; - } else { - textEl.textContent = "Download ready!"; - } + if (failed.length > 0) { + textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + } else { + textEl.textContent = "Download ready!"; + } - setTimeout(() => { - statusBox.style.display = "none"; - }, 5000); - }); - }); - }) - .fail(function(err) { - console.error("Error downloading submissions:", err); - textEl.textContent = "Error downloading!"; - }); - - // CODALAB.api.download_many_submissions(self.checked_submissions) - // .done(function(files) { - // console.log("Files returned by server:", files); - - // const zip = new JSZip(); - // const total = files.length; - // let completed = 0; - - // const fetchFiles = files.map(async file => { - // const response = await fetch(file.url); - // const blob = await response.blob(); - - // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); - - // // Update progress - // completed++; - // const percent = Math.floor((completed / total) * 100); - // progressEl.value = percent; - // // progressBar.progress({ percent: percent }); - // textEl.textContent = `${completed} / ${total} files (${percent}%)`; - // }); - - // Promise.all(fetchFiles).then(() => { - // textEl.textContent = "Generating bundle"; - // progressEl.style.display="none"; - - // zip.generateAsync({ type: 'blob' }).then(blob => { - // const link = document.createElement('a'); - // link.href = URL.createObjectURL(blob); - // link.download = 'bulk_submissions.zip'; - // document.body.appendChild(link); - // link.click(); - // document.body.removeChild(link); - - // textEl.textContent = "Download ready!"; - // setTimeout(() => { - // statusBox.style.display = "none"; - // }, 3000); - // }); - // }); - // }) - // .fail(function(err) { - // console.error("Error downloading submissions:", err); - // textEl.textContent = "Error downloading!"; - // }); + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Support both jQuery jqXHR (.done/.fail) and native Promise (.then/.catch) + if (req && typeof req.done === "function") { + // jQuery-style + req.done(function() { + // jQuery passes (data, textStatus, jqXHR) + // forward only the first argument to our handler + const args = Array.from(arguments); + handleSuccess(args[0]); + }).fail(handleError); + } else if (req && typeof req.then === "function") { + // native Promise + req.then(handleSuccess).catch(handleError); + } else { + console.error("download_many_submissions returned non-promise/non-jqXHR:", req); + handleError(new Error("Invalid request return type")); + } }; self.submission_handling = function () { From 0cae8f2115d77bcfcb56763d2ed45b0f8f198802 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 19 Sep 2025 15:22:33 +0200 Subject: [PATCH 19/21] it should work --- src/apps/api/views/submissions.py | 31 ++++++++++++++----- .../detail/submission_manager.tag | 2 -- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index e20b6f8b5..ad8cbfaca 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -18,7 +18,7 @@ from profiles.models import Organization, Membership from tasks.models import Task -from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer +from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer,SubmissionDetailSerializer from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard @@ -386,6 +386,8 @@ def re_run_many_submissions(self, request): @action(detail=False, methods=('POST',)) def download_many(self, request): + # logger.warning("Starting download Many.") + pks = request.data.get('pks') if not pks: return Response({"error": "`pks` field is required"}, status=400) @@ -394,17 +396,26 @@ def download_many(self, request): if not isinstance(pks, list): return Response({"error": "`pks` must be a list"}, status=400) + + # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", "data" - ) - # .only("id","owner", "data__data_file") + )#.only("id","owner", "data__data_file") + + # logger.warning("Submissions fetch from DB.") + + # .prefetch_related("phase__competition__collaborators") - if submissions.count() != len(pks): + + # .count() + if len(list(submissions)) != len(pks): + # if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) + # NH : should should create a function for this ? # Check permissions if not request.user.is_authenticated: @@ -427,13 +438,19 @@ def download_many(self, request): ) files = [] + + # logger.warning("Submissions security passed") + + + for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" - - url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] + # url = sub.data.data_file.url + url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] + # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] files.append({"name": short_name, "url": url}) - + return Response(files) @action(detail=True, methods=('GET',)) diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 148980fdc..39bcacb7c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -715,7 +715,6 @@ let completed = 0; const failed = []; - // concurrency limit (tune this; 5-10 is usually a good start) const limit = 5; const queue = files.slice(); // clone const running = []; @@ -754,7 +753,6 @@ await Promise.race(running); } } - // wait for remaining ones await Promise.all(running); // Add failed.txt if necessary From fd84799cef329e760f8802592d82e61dbbf57372 Mon Sep 17 00:00:00 2001 From: Nicolas HOMBERG Date: Fri, 19 Sep 2025 15:46:49 +0200 Subject: [PATCH 20/21] cleaned comments --- src/apps/api/views/submissions.py | 10 ---------- src/static/js/ours/client.js | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index ad8cbfaca..496bf4ab1 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -386,8 +386,6 @@ def re_run_many_submissions(self, request): @action(detail=False, methods=('POST',)) def download_many(self, request): - # logger.warning("Starting download Many.") - pks = request.data.get('pks') if not pks: return Response({"error": "`pks` field is required"}, status=400) @@ -405,12 +403,8 @@ def download_many(self, request): "data" )#.only("id","owner", "data__data_file") - # logger.warning("Submissions fetch from DB.") - - # .prefetch_related("phase__competition__collaborators") - # .count() if len(list(submissions)) != len(pks): # if submissions.count() != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) @@ -439,10 +433,6 @@ def download_many(self, request): files = [] - # logger.warning("Submissions security passed") - - - for sub in submissions: file_path = sub.data.data_file.name.split('/')[-1] short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 9fbd37251..f43bfaaa2 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -135,17 +135,6 @@ CODALAB.api = { ); }, - // download_many_submissions: function (pks) { - // return CODALAB.api.request('POST', `${URLS.API}submissions/download_many/`, { - // body: JSON.stringify({ pks: pks }), - // headers: { - // "Content-Type": "application/json" - // } - // }); - // const params = new URLSearchParams({ pks: JSON.stringify(pks) }); - // const url = `${URLS.API}submissions/download_many/?${params}`; - // return CODALAB.api.request('GET', url) - // }, /*--------------------------------------------------------------------- Leaderboards ---------------------------------------------------------------------*/ From 53a0f73e1acba3b54017966d24bb17c835d1d314 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 23 Sep 2025 10:51:45 +0200 Subject: [PATCH 21/21] Flake8 fixes --- src/apps/api/views/submissions.py | 21 ++++++--------------- src/apps/competitions/tasks.py | 4 ---- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 496bf4ab1..6391bace8 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -14,11 +14,10 @@ from rest_framework.viewsets import ModelViewSet from rest_framework_csv import renderers from django.core.files.base import ContentFile -from django.http import StreamingHttpResponse from profiles.models import Organization, Membership from tasks.models import Task -from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer,SubmissionDetailSerializer +from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer, SubmissionDetailSerializer from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard @@ -382,8 +381,6 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - - @action(detail=False, methods=('POST',)) def download_many(self, request): pks = request.data.get('pks') @@ -393,24 +390,18 @@ def download_many(self, request): # pks is already parsed as a list if JSON was sent properly if not isinstance(pks, list): return Response({"error": "`pks` must be a list"}, status=400) - - # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", "phase", "data" - )#.only("id","owner", "data__data_file") - - # .prefetch_related("phase__competition__collaborators") + ) - if len(list(submissions)) != len(pks): - # if submissions.count() != len(pks): + if len(list(submissions)) != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) - - # NH : should should create a function for this ? + # Nicolas Homberg : should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -440,9 +431,9 @@ def download_many(self, request): url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] files.append({"name": short_name, "url": url}) - + return Response(files) - + @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 631aa0815..4c5e495b8 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -19,10 +19,6 @@ from django.utils.timezone import now from rest_framework.exceptions import ValidationError -from urllib.request import urlopen -from contextlib import closing -from urllib.error import ContentTooShortError - from celery_config import app from competitions.models import Submission, CompetitionCreationTaskStatus, SubmissionDetails, Competition, \ CompetitionDump, Phase