From 30f54bf7737076ab3d5159d42efeefe85a59ba20 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Mon, 17 Dec 2018 18:44:00 +1100 Subject: [PATCH 1/7] Added method to fetch project versions using GitHub API, removed hardcoded versions in tests. --- l2tdevtools/download_helpers/github.py | 42 ++++++++++++++++++++++++++ tests/download_helpers/github.py | 27 ++++++++++++----- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/l2tdevtools/download_helpers/github.py b/l2tdevtools/download_helpers/github.py index cfcb86ac..05f00da1 100644 --- a/l2tdevtools/download_helpers/github.py +++ b/l2tdevtools/download_helpers/github.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals +import json import re from l2tdevtools.download_helpers import project @@ -76,6 +77,47 @@ def _GetAvailableVersions(self, version_strings): return available_versions + def GetLatestVersionWithAPI(self, version_definition): + """Uses the GitHub API to retrieve the latest version number for a project. + + This method is intended for use in tests only, due to the API rate-limit. + + Args: + version_definition (ProjectVersionDefinition): project version definition + or None if not set. + + Returns: + str: latest version number or None if not available. + """ + earliest_version = None + latest_version = None + + if version_definition: + earliest_version = version_definition.GetEarliestVersion() + if earliest_version and earliest_version[0] == '==': + return '.'.join(earliest_version[1:]) + + latest_version = version_definition.GetLatestVersion() + + github_url = 'https://api.github.com/repos/{0:s}/{1:s}/releases'.format( + self._organization, self._repository) + page_content = self.DownloadPageContent(github_url) + if not page_content: + return None + + api_response = json.loads(page_content) + release_names = [release['name'] for release in api_response] + + expression_string = '({0:s})'.format('|'.join(self._VERSION_EXPRESSIONS)) + versions = [] + for release in release_names: + version_strings = re.findall(expression_string, release) + versions.extend(version_strings) + + available_versions = self._GetAvailableVersions(versions) + return self._GetLatestVersion( + earliest_version, latest_version, available_versions) + def GetLatestVersion(self, project_name, version_definition): """Retrieves the latest version number for a given project name. diff --git a/tests/download_helpers/github.py b/tests/download_helpers/github.py index 7e7a5bf5..a3ead72e 100644 --- a/tests/download_helpers/github.py +++ b/tests/download_helpers/github.py @@ -22,6 +22,8 @@ class DocoptGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): _PROJECT_ORGANIZATION = 'docopt' _PROJECT_NAME = 'docopt' + # Hardcoded version to check parsing of the GitHub page, as the GitHub API + # does not return release information for this project. _PROJECT_VERSION = '0.6.2' def testGetLatestVersion(self): @@ -76,21 +78,26 @@ def testGetLatestVersion(self): download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version_api = download_helper.GetLatestVersionWithAPI( + self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + self.assertEqual(latest_version, latest_version_api) def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + project_version = download_helper.GetLatestVersionWithAPI( + self._PROJECT_NAME, None) + download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + self._PROJECT_NAME, project_version) expected_download_url = ( 'https://github.com/{0:s}/{1:s}/releases/download/{3:s}/' '{1:s}-{2:s}-{3:s}.tar.gz').format( self._PROJECT_ORGANIZATION, self._PROJECT_NAME, - self._PROJECT_STATUS, self._PROJECT_VERSION) + self._PROJECT_STATUS, project_version) self.assertEqual(download_url, expected_download_url) @@ -116,29 +123,33 @@ class Log2TimelineGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): _PROJECT_ORGANIZATION = 'log2timeline' _PROJECT_NAME = 'dfvfs' - # Hard-coded version to check parsing of GitHub page. - _PROJECT_VERSION = '20190128' def testGetLatestVersion(self): """Tests the GetLatestVersion functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version_api = download_helper.GetLatestVersionWithAPI( + self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + + self.assertEqual(latest_version, latest_version_api) def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + project_version = download_helper.GetLatestVersionWithAPI( + self._PROJECT_NAME, None) + download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + self._PROJECT_NAME, project_version) expected_download_url = ( 'https://github.com/{0:s}/{1:s}/releases/download/{2:s}/' '{1:s}-{2:s}.tar.gz').format( self._PROJECT_ORGANIZATION, self._PROJECT_NAME, - self._PROJECT_VERSION) + project_version) self.assertEqual(download_url, expected_download_url) From fe5cac1d9d90213db31467881a13543cafee3583 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Tue, 18 Dec 2018 16:53:35 +1100 Subject: [PATCH 2/7] Added method to fetch project versions using GitHub API, removed hardcoded versions in tests. --- tests/download_helpers/github.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/download_helpers/github.py b/tests/download_helpers/github.py index a3ead72e..fcd27efd 100644 --- a/tests/download_helpers/github.py +++ b/tests/download_helpers/github.py @@ -78,8 +78,7 @@ def testGetLatestVersion(self): download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - latest_version_api = download_helper.GetLatestVersionWithAPI( - self._PROJECT_NAME, None) + latest_version_api = download_helper.GetLatestVersionWithAPI(None) self.assertEqual(latest_version, latest_version_api) @@ -87,8 +86,7 @@ def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - project_version = download_helper.GetLatestVersionWithAPI( - self._PROJECT_NAME, None) + project_version = download_helper.GetLatestVersionWithAPI(None) download_url = download_helper.GetDownloadURL( self._PROJECT_NAME, project_version) @@ -129,8 +127,7 @@ def testGetLatestVersion(self): download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - latest_version_api = download_helper.GetLatestVersionWithAPI( - self._PROJECT_NAME, None) + latest_version_api = download_helper.GetLatestVersionWithAPI(None) self.assertEqual(latest_version, latest_version_api) @@ -139,8 +136,7 @@ def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - project_version = download_helper.GetLatestVersionWithAPI( - self._PROJECT_NAME, None) + project_version = download_helper.GetLatestVersionWithAPI(None) download_url = download_helper.GetDownloadURL( self._PROJECT_NAME, project_version) From 4b4f1473560f79195dad2f8d33693750865ca779 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Tue, 18 Dec 2018 16:54:19 +1100 Subject: [PATCH 3/7] Added method to fetch project versions using GitHub API, removed hardcoded versions in tests. --- l2tdevtools/download_helpers/github.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/l2tdevtools/download_helpers/github.py b/l2tdevtools/download_helpers/github.py index 05f00da1..cc86c70d 100644 --- a/l2tdevtools/download_helpers/github.py +++ b/l2tdevtools/download_helpers/github.py @@ -108,11 +108,16 @@ def GetLatestVersionWithAPI(self, version_definition): api_response = json.loads(page_content) release_names = [release['name'] for release in api_response] - expression_string = '({0:s})'.format('|'.join(self._VERSION_EXPRESSIONS)) + version_expressions = [ + '({0:s})$'.format(version_expression) + for version_expression + in self._VERSION_EXPRESSIONS] + versions = [] for release in release_names: - version_strings = re.findall(expression_string, release) - versions.extend(version_strings) + for version_expression in version_expressions: + version_strings = re.findall(version_expression, release) + versions.extend(version_strings) available_versions = self._GetAvailableVersions(versions) return self._GetLatestVersion( From 6e14b5416849750655e041a4d799cb85e3aaff77 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Tue, 18 Dec 2018 18:17:52 +1100 Subject: [PATCH 4/7] Added method to fetch project versions using GitHub API, removed hardcoded versions in tests. --- tests/download_helpers/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/download_helpers/github.py b/tests/download_helpers/github.py index fcd27efd..aeb212aa 100644 --- a/tests/download_helpers/github.py +++ b/tests/download_helpers/github.py @@ -86,7 +86,7 @@ def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) - project_version = download_helper.GetLatestVersionWithAPI(None) + project_version = download_helper.GetLatestVersionWithAPI(None) download_url = download_helper.GetDownloadURL( self._PROJECT_NAME, project_version) From a2b7af85b79fbdd28322caa0e95e5f9605060511 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Wed, 19 Dec 2018 12:08:28 +1100 Subject: [PATCH 5/7] Added method to fetch project versions using PyPi API, removed hardcoded versions in tests. --- l2tdevtools/download_helpers/pypi.py | 33 ++++++++++++++++++++++++++++ tests/download_helpers/pypi.py | 3 +++ 2 files changed, 36 insertions(+) diff --git a/l2tdevtools/download_helpers/pypi.py b/l2tdevtools/download_helpers/pypi.py index 03c8a76b..29124e0a 100644 --- a/l2tdevtools/download_helpers/pypi.py +++ b/l2tdevtools/download_helpers/pypi.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals +import json import re # pylint: disable=wrong-import-position @@ -78,6 +79,38 @@ def _GetAvailableVersions(self, version_strings): return available_versions + def GetLatestVersionWithAPI(self, project_name, version_definition): + """Retrieves the latest version for a given project using the PyPi API. + + Args: + project_name (str): name of the project. + version_definition (ProjectVersionDefinition): project version definition + or None. + + Returns: + str: latest version number or None if not available. + """ + earliest_version = None + latest_version = None + + if version_definition: + earliest_version = version_definition.GetEarliestVersion() + if earliest_version and earliest_version[0] == '==': + return '.'.join(earliest_version[1:]) + + latest_version = version_definition.GetLatestVersion() + + pypi_url = 'https://pypi.org/pypi/{0:s}/json'.format(project_name) + page_content = self.DownloadPageContent(pypi_url) + + api_data = json.loads(page_content) + releases = api_data.get('releases', {}) + version_strings = releases.keys() + available_versions = self._GetAvailableVersions(version_strings) + + return self._GetLatestVersion( + earliest_version, latest_version, available_versions) + # pylint: disable=unused-argument def GetLatestVersion(self, project_name, version_definition): """Retrieves the latest version number for a given project name. diff --git a/tests/download_helpers/pypi.py b/tests/download_helpers/pypi.py index bd33d204..9f88343d 100644 --- a/tests/download_helpers/pypi.py +++ b/tests/download_helpers/pypi.py @@ -30,6 +30,9 @@ def testGetLatestVersion(self): latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version = download_helper.GetLatestVersionWithAPI( + self._PROJECT_NAME, None) + self.assertEqual(latest_version, self._PROJECT_VERSION) def testGetDownloadURL(self): From fc22c888cf1ea3577f06a08ab30e60edeabf682b Mon Sep 17 00:00:00 2001 From: Daniel White Date: Fri, 18 Jan 2019 11:32:04 +0100 Subject: [PATCH 6/7] Added method to fetch project versions using PyPi API, removed hardcoded versions in tests. --- l2tdevtools/download_helpers/github.py | 3 +-- l2tdevtools/download_helpers/interface.py | 27 +++++++++++++++++++++++ tests/download_helpers/pypi.py | 4 ++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/l2tdevtools/download_helpers/github.py b/l2tdevtools/download_helpers/github.py index cc86c70d..28e324c1 100644 --- a/l2tdevtools/download_helpers/github.py +++ b/l2tdevtools/download_helpers/github.py @@ -102,8 +102,7 @@ def GetLatestVersionWithAPI(self, version_definition): github_url = 'https://api.github.com/repos/{0:s}/{1:s}/releases'.format( self._organization, self._repository) page_content = self.DownloadPageContent(github_url) - if not page_content: - return None + api_response = json.loads(page_content) release_names = [release['name'] for release in api_response] diff --git a/l2tdevtools/download_helpers/interface.py b/l2tdevtools/download_helpers/interface.py index fd097a5e..77c9ba63 100644 --- a/l2tdevtools/download_helpers/interface.py +++ b/l2tdevtools/download_helpers/interface.py @@ -103,3 +103,30 @@ def DownloadPageContent(self, download_url, encoding='utf-8'): self._cached_url = download_url return self._cached_page_content + + def DownloadAPIPageContent(self, download_url, encoding='utf-8'): + """Download content from a github API url""" + if not download_url: + return None + + if self._cached_url != download_url: + try: + url_object = urllib_request.urlopen(download_url) + except urllib_error.URLError as exception: + logging.warning( + 'Unable to download URL: {0:s} with error: {1!s}'.format( + download_url, exception)) + return None + + if url_object.code != 403: + return None + + page_content = url_object.read() + + if encoding and isinstance(page_content, py2to3.BYTES_TYPE): + page_content = page_content.decode(encoding) + + self._cached_page_content = page_content + self._cached_url = download_url + + return self._cached_page_content diff --git a/tests/download_helpers/pypi.py b/tests/download_helpers/pypi.py index 9f88343d..0a98b5e2 100644 --- a/tests/download_helpers/pypi.py +++ b/tests/download_helpers/pypi.py @@ -30,10 +30,10 @@ def testGetLatestVersion(self): latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - latest_version = download_helper.GetLatestVersionWithAPI( + latest_version_with_api = download_helper.GetLatestVersionWithAPI( self._PROJECT_NAME, None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + self.assertEqual(latest_version, latest_version_with_api) def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" From 1927d76954f029fa96cb3171e8c265b30d77cbb1 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Sat, 13 Apr 2019 16:39:39 +0200 Subject: [PATCH 7/7] Reverted pypi changes --- l2tdevtools/download_helpers/pypi.py | 33 ---------------------------- tests/download_helpers/pypi.py | 5 +---- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/l2tdevtools/download_helpers/pypi.py b/l2tdevtools/download_helpers/pypi.py index 29124e0a..03c8a76b 100644 --- a/l2tdevtools/download_helpers/pypi.py +++ b/l2tdevtools/download_helpers/pypi.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals -import json import re # pylint: disable=wrong-import-position @@ -79,38 +78,6 @@ def _GetAvailableVersions(self, version_strings): return available_versions - def GetLatestVersionWithAPI(self, project_name, version_definition): - """Retrieves the latest version for a given project using the PyPi API. - - Args: - project_name (str): name of the project. - version_definition (ProjectVersionDefinition): project version definition - or None. - - Returns: - str: latest version number or None if not available. - """ - earliest_version = None - latest_version = None - - if version_definition: - earliest_version = version_definition.GetEarliestVersion() - if earliest_version and earliest_version[0] == '==': - return '.'.join(earliest_version[1:]) - - latest_version = version_definition.GetLatestVersion() - - pypi_url = 'https://pypi.org/pypi/{0:s}/json'.format(project_name) - page_content = self.DownloadPageContent(pypi_url) - - api_data = json.loads(page_content) - releases = api_data.get('releases', {}) - version_strings = releases.keys() - available_versions = self._GetAvailableVersions(version_strings) - - return self._GetLatestVersion( - earliest_version, latest_version, available_versions) - # pylint: disable=unused-argument def GetLatestVersion(self, project_name, version_definition): """Retrieves the latest version number for a given project name. diff --git a/tests/download_helpers/pypi.py b/tests/download_helpers/pypi.py index 0a98b5e2..bd33d204 100644 --- a/tests/download_helpers/pypi.py +++ b/tests/download_helpers/pypi.py @@ -30,10 +30,7 @@ def testGetLatestVersion(self): latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) - latest_version_with_api = download_helper.GetLatestVersionWithAPI( - self._PROJECT_NAME, None) - - self.assertEqual(latest_version, latest_version_with_api) + self.assertEqual(latest_version, self._PROJECT_VERSION) def testGetDownloadURL(self): """Tests the GetDownloadURL functions."""