From 1f6d1ba6b9826c3b7f916449671d4b81c61a0962 Mon Sep 17 00:00:00 2001 From: sambattalio Date: Thu, 11 Jul 2019 10:59:55 -0400 Subject: [PATCH 1/2] Fix: Change disabled repository to only warning message --- CHANGELOG.md | 2 +- tap_gitlab/__init__.py | 30 ++++++++++++++++++------------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f57a3d3..8e05009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## 0.5.1 - * Update version of `requests` to `0.20.0` in response to CVE 2018-18074 + * Update version of `requests` to `2.20.0` in response to CVE 2018-18074 ## 0.5.0 * Added support for groups and group milestones [#9](https://github.com/singer-io/tap-gitlab/pull/9) diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index d8d1277..698e018 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -91,7 +91,7 @@ def get_start(entity): @backoff.on_exception(backoff.expo, (requests.exceptions.RequestException), max_tries=5, - giveup=lambda e: e.response is not None and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long + giveup=lambda e: e.response is not None and e.response.status_code != 404 and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long factor=2) def request(url, params=None): params = params or {} @@ -105,10 +105,14 @@ def request(url, params=None): LOGGER.info("GET {}".format(req.url)) resp = SESSION.send(req) - if resp.status_code >= 400: - LOGGER.critical( - "Error making request to GitLab API: GET {} [{} - {}]".format( - req.url, resp.status_code, resp.content)) + if resp.status_code == 404 and b'404 Repository Not Found' in resp.content: + # Don't exit program if repository disabled (404) + LOGGER.warning("Disabled repository: GET {} [{} - {}]".format( + req.url, resp.status_code, resp.content)) + return -1 + elif resp.status_code >= 400: + LOGGER.critical("Error making request to GitLab API: GET {} [{} - {}]".format( + req.url, resp.status_code, resp.content)) sys.exit(1) return resp @@ -117,17 +121,19 @@ def request(url, params=None): def gen_request(url): params = {'page': 1} resp = request(url, params) - last_page = int(resp.headers.get('X-Total-Pages', 1)) + + if resp != -1: + last_page = int(resp.headers.get('X-Total-Pages', 1)) - for row in resp.json(): - yield row - - for page in range(2, last_page + 1): - params['page'] = page - resp = request(url, params) for row in resp.json(): yield row + for page in range(2, last_page + 1): + params['page'] = page + resp = request(url, params) + for row in resp.json(): + yield row + def format_timestamp(data, typ, schema): result = data if typ == 'string' and schema.get('format') == 'date-time': From 0e2184068374ea1c480df847d74ef9768c989583 Mon Sep 17 00:00:00 2001 From: sambattalio Date: Thu, 11 Jul 2019 11:06:38 -0400 Subject: [PATCH 2/2] Changed giveup lambda to original --- tap_gitlab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index 698e018..dc3ff7f 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -91,7 +91,7 @@ def get_start(entity): @backoff.on_exception(backoff.expo, (requests.exceptions.RequestException), max_tries=5, - giveup=lambda e: e.response is not None and e.response.status_code != 404 and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long + giveup=lambda e: e.response is not None and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long factor=2) def request(url, params=None): params = params or {}