From 8c1714e2b8ab679adb7662801e30efa5656ed9b6 Mon Sep 17 00:00:00 2001 From: MarCnu Date: Tue, 12 Aug 2014 17:27:36 +0200 Subject: [PATCH 1/3] AUTHORS update --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 96bca29d4a25..6666345cc51e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -169,3 +169,4 @@ Clinton Blackburn Dennis Jen Filippo Valsorda Ivica Ceraj +Marceau Cnudde From 17d4b2ac2f71eead1337609670c5aa24acb8f143 Mon Sep 17 00:00:00 2001 From: MarCnu Date: Fri, 1 Aug 2014 18:10:47 +0200 Subject: [PATCH 2/3] Add HTTP_RANGE compatibility for ContentStore file streaming Currently, users can only download ContentStore files from first byte to last byte. With this change, when a request to the ContentStore includes the HTTP "Range" parameter, it is parsed and StaticContentStream will stream the requested bytes. This change makes possible to stream video files (.mp4 especially) from the ContentStore. Users can now seek a specific time in the video without loading all the file. This is useful for courses with a limited number of students that doesn't require a dedicated video server. --- common/djangoapps/contentserver/middleware.py | 43 +++++++++++- common/djangoapps/contentserver/tests/test.py | 70 ++++++++++++++++++- .../xmodule/xmodule/contentstore/content.py | 19 ++++- 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index e711fd2b6612..0e508c413e00 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -75,7 +75,48 @@ def process_request(self, request): if if_modified_since == last_modified_at_str: return HttpResponseNotModified() - response = HttpResponse(content.stream_data(), content_type=content.content_type) + # *** File streaming within a byte range *** + # If a Range is provided, parse Range attribute of the request + # Add Content-Range in the response if Range is structurally correct + # Request -> Range attribute structure: "Range: bytes=first-[last]" + # Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength" + response = None + if request.META.get('HTTP_RANGE'): + range_header = request.META['HTTP_RANGE'] + if '=' in range_header: + unit, range = range_header.split('=') + if unit == 'bytes' and '-' in range: + first, last = range.split('-') + try: + first = int(first) + except ValueError: + first = 0 + try: + last = int(last) + except ValueError: + last = content.length - 1 + + if 0 <= first <= last < content.length: + # Valid Range attribute + response = HttpResponse(content.stream_data_in_range(first, last)) + response['Content-Range'] = 'bytes {first}-{last}/{length}'.format( + first=first, last=last, length=content.length + ) + response['Content-Length'] = str(last - first + 1) + response.status_code = 206 # HTTP_206_PARTIAL_CONTENT + if not response: + # Malformed Range attribute + response = HttpResponse() + response.status_code = 400 # HTTP_400_BAD_REQUEST + return response + + else: + # No Range attribute + response = HttpResponse(content.stream_data()) + response['Content-Length'] = content.length + + response['Accept-Ranges'] = 'bytes' + response['Content-Type'] = content.content_type response['Last-Modified'] = last_modified_at_str return response diff --git a/common/djangoapps/contentserver/tests/test.py b/common/djangoapps/contentserver/tests/test.py index 465b49709af5..ce8a45a98ec6 100644 --- a/common/djangoapps/contentserver/tests/test.py +++ b/common/djangoapps/contentserver/tests/test.py @@ -48,12 +48,12 @@ def setUp(self): # A locked asset self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt') self.url_locked = self.locked_asset.to_deprecated_string() + self.contentstore.set_attr(self.locked_asset, 'locked', True) # An unlocked asset self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt') self.url_unlocked = self.unlocked_asset.to_deprecated_string() - - self.contentstore.set_attr(self.locked_asset, 'locked', True) + self.length_unlocked = self.contentstore.get_attr(self.unlocked_asset, 'length') def test_unlocked_asset(self): """ @@ -101,3 +101,69 @@ def test_locked_asset_staff(self): resp = self.client.get(self.url_locked) self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 + def test_range_request_full_file(self): + """ + Test that a range request from byte 0 to last outputs partial content status code and valid Content-Range. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-') + self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT + self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format( + first=0, last=self.length_unlocked-1, length=self.length_unlocked + ) + ) + + def test_range_request_partial_file(self): + """ + Test that a range request for a partial file outputs partial content status code and valid Content-Range. + firstByte and lastByte are chosen to be simple but non trivial values. + """ + firstByte = self.length_unlocked / 4 + lastByte = self.length_unlocked / 2 + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format( + first=firstByte, last=lastByte + ) + ) + self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT + self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format( + first=firstByte, last=lastByte, length=lastByte - firstByte + 1 + ) + ) + + def test_range_request_malformed_missing_equal(self): + """ + Test that a range request with malformed Range (missing '=') outputs status 400. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes 0-') + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + + def test_range_request_malformed_missing_minus(self): + """ + Test that a range request with malformed Range (missing '-') outputs status 400. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0') + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_invalid_range(self): + """ + Test that a range request with malformed Range (firstByte > lastByte) outputs status 400. + """ + firstByte = self.length_unlocked / 2 + lastByte = self.length_unlocked / 4 + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format( + first=firstByte, last=lastByte + ) + ) + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_out_of_bounds(self): + """ + Test that a range request with malformed Range (lastByte == totalLength, offset by 1 error) outputs status 400. + """ + lastByte = self.length_unlocked + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-{last}'.format( + last=lastByte + ) + ) + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py index 73ec3d23c4a7..1d8573b15369 100644 --- a/common/lib/xmodule/xmodule/contentstore/content.py +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -4,6 +4,8 @@ XASSET_THUMBNAIL_TAIL_NAME = '.jpg' +STREAM_DATA_CHUNK_SIZE = 1024 + import os import logging import StringIO @@ -162,11 +164,26 @@ def __init__(self, loc, name, content_type, stream, last_modified_at=None, thumb def stream_data(self): while True: - chunk = self._stream.read(1024) + chunk = self._stream.read(STREAM_DATA_CHUNK_SIZE) if len(chunk) == 0: break yield chunk + def stream_data_in_range(self, first_byte, last_byte): + """ + Stream the data between first_byte and last_byte (included) + """ + self._stream.seek(first_byte) + position = first_byte + while True: + if last_byte < position + STREAM_DATA_CHUNK_SIZE - 1: + chunk = self._stream.read(last_byte - position + 1) + yield chunk + break + chunk = self._stream.read(STREAM_DATA_CHUNK_SIZE) + position += STREAM_DATA_CHUNK_SIZE + yield chunk + def close(self): self._stream.close() From 86a8d6a2eec612a87f785de1ee3d7bd13dab982e Mon Sep 17 00:00:00 2001 From: David Baumgold Date: Wed, 13 Aug 2014 12:01:59 -0400 Subject: [PATCH 3/3] Use mock.patch in test_range_request_full_file --- common/djangoapps/contentserver/tests/test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/djangoapps/contentserver/tests/test.py b/common/djangoapps/contentserver/tests/test.py index ce8a45a98ec6..3e8470880da9 100644 --- a/common/djangoapps/contentserver/tests/test.py +++ b/common/djangoapps/contentserver/tests/test.py @@ -4,6 +4,7 @@ import copy import logging from uuid import uuid4 +from mock import patch from django.conf import settings from django.test.client import Client @@ -101,10 +102,14 @@ def test_locked_asset_staff(self): resp = self.client.get(self.url_locked) self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 - def test_range_request_full_file(self): + @patch("common.djangoapps.contentserver.middleware.contentstore") + def test_range_request_full_file(self, mock_contentstore): """ Test that a range request from byte 0 to last outputs partial content status code and valid Content-Range. """ + mock_content = mock_contentstore.return_value.find.return_value + mock_content.length = 5 + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-') self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format( @@ -135,7 +140,7 @@ def test_range_request_malformed_missing_equal(self): """ resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes 0-') self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST - + def test_range_request_malformed_missing_minus(self): """ @@ -166,4 +171,4 @@ def test_range_request_malformed_out_of_bounds(self): ) ) self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST - +