From d248b3b21b12d318ca25ad830f351208e0911349 Mon Sep 17 00:00:00 2001 From: Chris Opperwall Date: Fri, 12 Jan 2018 09:55:23 -0800 Subject: [PATCH 1/2] Use generator for get_all_serial get_all_serial is used to iterate through a paginated response from shipwire's API. Previously it would add each batch of items to a list and then return that list once there were no more results to get. This changes get_all_serial to be a generator, so that the program only has to keep a single batch's worth of items in memory at a time. This is especially helpful for grabbing large datasets from shipwire, because you can begin processing items from shipwire while there are still items to be requested. --- README.md | 2 +- shipwire/responses.py | 8 ++++---- shipwire/tests/test_responses.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index da55edc..48eea26 100755 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ response.next # returns a URL to the next group of items in the pagination response.offset # returns the nubmer of items offset by the current request response.items # returns a list of items in the current group response.limit # returns a count of the current group -response.all() # returns a list of all the items in the entire selection. Please note that this method can be time consuming and lead to timeout errors by the Shipwire API. +response.all() # returns a generator of all the items in the entire selection. Please note that this method can be time consuming and lead to timeout errors by the Shipwire API. ``` ##### Unsuccessful requests: diff --git a/shipwire/responses.py b/shipwire/responses.py index 502e501..63c45f0 100644 --- a/shipwire/responses.py +++ b/shipwire/responses.py @@ -49,16 +49,16 @@ def _get_all_serial(self): # loop over all items with previous and next next_uri = self.__next__ req = self.response.request - items = self.items + + for item in self.items: + yield item while next_uri: resp = requests.request(req.method, next_uri, auth=self.shipwire.auth) list_resp = ListResponse(resp, self.shipwire) - items.extend(list_resp.items) + yield from list_resp.items next_uri = list_resp.__next__ - return items - class CreateResponse(ListResponse): pass diff --git a/shipwire/tests/test_responses.py b/shipwire/tests/test_responses.py index 81065e4..a9e58a4 100644 --- a/shipwire/tests/test_responses.py +++ b/shipwire/tests/test_responses.py @@ -52,7 +52,7 @@ def test_all_calls_until_end(self, mock): mock.side_effect = [fake_list_response([], next=p) for p in subsequent] sw_response = responses.ListResponse(initial_response, MagicMock()) - sw_response.all_serial() + list(sw_response.all_serial()) self.assertEqual(len(subsequent), mock.call_count) @@ -65,4 +65,4 @@ def test_all_returns_all_items(self, mock): for i, p in enumerate(subsequent)] sw_response = responses.ListResponse(initial_response, MagicMock()) - self.assertEqual([0, 1, 2, 3, 4], sw_response.all_serial()) + self.assertEqual([0, 1, 2, 3, 4], list(sw_response.all_serial())) From c3b7e7a809456f566815b0cd29e86041b91fa8c5 Mon Sep 17 00:00:00 2001 From: Chris Opperwall Date: Mon, 15 Jan 2018 21:04:13 -0800 Subject: [PATCH 2/2] Remove yield from for python2 compatability --- shipwire/responses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shipwire/responses.py b/shipwire/responses.py index 63c45f0..29e6df5 100644 --- a/shipwire/responses.py +++ b/shipwire/responses.py @@ -56,7 +56,8 @@ def _get_all_serial(self): while next_uri: resp = requests.request(req.method, next_uri, auth=self.shipwire.auth) list_resp = ListResponse(resp, self.shipwire) - yield from list_resp.items + for item in list_resp.items: + yield item next_uri = list_resp.__next__