diff --git a/l2tdevtools/download_helpers/github.py b/l2tdevtools/download_helpers/github.py index cfcb86ac..28e324c1 100644 --- a/l2tdevtools/download_helpers/github.py +++ b/l2tdevtools/download_helpers/github.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals +import json import re from l2tdevtools.download_helpers import project @@ -76,6 +77,51 @@ def _GetAvailableVersions(self, version_strings): return available_versions + def GetLatestVersionWithAPI(self, version_definition): + """Uses the GitHub API to retrieve the latest version number for a project. + + This method is intended for use in tests only, due to the API rate-limit. + + Args: + version_definition (ProjectVersionDefinition): project version definition + or None if not set. + + Returns: + str: latest version number or None if not available. + """ + earliest_version = None + latest_version = None + + if version_definition: + earliest_version = version_definition.GetEarliestVersion() + if earliest_version and earliest_version[0] == '==': + return '.'.join(earliest_version[1:]) + + latest_version = version_definition.GetLatestVersion() + + github_url = 'https://api.github.com/repos/{0:s}/{1:s}/releases'.format( + self._organization, self._repository) + page_content = self.DownloadPageContent(github_url) + + + api_response = json.loads(page_content) + release_names = [release['name'] for release in api_response] + + version_expressions = [ + '({0:s})$'.format(version_expression) + for version_expression + in self._VERSION_EXPRESSIONS] + + versions = [] + for release in release_names: + for version_expression in version_expressions: + version_strings = re.findall(version_expression, release) + versions.extend(version_strings) + + available_versions = self._GetAvailableVersions(versions) + return self._GetLatestVersion( + earliest_version, latest_version, available_versions) + def GetLatestVersion(self, project_name, version_definition): """Retrieves the latest version number for a given project name. diff --git a/l2tdevtools/download_helpers/interface.py b/l2tdevtools/download_helpers/interface.py index fd097a5e..77c9ba63 100644 --- a/l2tdevtools/download_helpers/interface.py +++ b/l2tdevtools/download_helpers/interface.py @@ -103,3 +103,30 @@ def DownloadPageContent(self, download_url, encoding='utf-8'): self._cached_url = download_url return self._cached_page_content + + def DownloadAPIPageContent(self, download_url, encoding='utf-8'): + """Download content from a github API url""" + if not download_url: + return None + + if self._cached_url != download_url: + try: + url_object = urllib_request.urlopen(download_url) + except urllib_error.URLError as exception: + logging.warning( + 'Unable to download URL: {0:s} with error: {1!s}'.format( + download_url, exception)) + return None + + if url_object.code != 403: + return None + + page_content = url_object.read() + + if encoding and isinstance(page_content, py2to3.BYTES_TYPE): + page_content = page_content.decode(encoding) + + self._cached_page_content = page_content + self._cached_url = download_url + + return self._cached_page_content diff --git a/tests/download_helpers/github.py b/tests/download_helpers/github.py index 7e7a5bf5..aeb212aa 100644 --- a/tests/download_helpers/github.py +++ b/tests/download_helpers/github.py @@ -22,6 +22,8 @@ class DocoptGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): _PROJECT_ORGANIZATION = 'docopt' _PROJECT_NAME = 'docopt' + # Hardcoded version to check parsing of the GitHub page, as the GitHub API + # does not return release information for this project. _PROJECT_VERSION = '0.6.2' def testGetLatestVersion(self): @@ -76,21 +78,24 @@ def testGetLatestVersion(self): download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version_api = download_helper.GetLatestVersionWithAPI(None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + self.assertEqual(latest_version, latest_version_api) def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + project_version = download_helper.GetLatestVersionWithAPI(None) + download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + self._PROJECT_NAME, project_version) expected_download_url = ( 'https://github.com/{0:s}/{1:s}/releases/download/{3:s}/' '{1:s}-{2:s}-{3:s}.tar.gz').format( self._PROJECT_ORGANIZATION, self._PROJECT_NAME, - self._PROJECT_STATUS, self._PROJECT_VERSION) + self._PROJECT_STATUS, project_version) self.assertEqual(download_url, expected_download_url) @@ -116,29 +121,31 @@ class Log2TimelineGitHubReleasesDownloadHelperTest(test_lib.BaseTestCase): _PROJECT_ORGANIZATION = 'log2timeline' _PROJECT_NAME = 'dfvfs' - # Hard-coded version to check parsing of GitHub page. - _PROJECT_VERSION = '20190128' def testGetLatestVersion(self): """Tests the GetLatestVersion functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) latest_version = download_helper.GetLatestVersion(self._PROJECT_NAME, None) + latest_version_api = download_helper.GetLatestVersionWithAPI(None) - self.assertEqual(latest_version, self._PROJECT_VERSION) + + self.assertEqual(latest_version, latest_version_api) def testGetDownloadURL(self): """Tests the GetDownloadURL functions.""" download_helper = github.GitHubReleasesDownloadHelper(self._DOWNLOAD_URL) + project_version = download_helper.GetLatestVersionWithAPI(None) + download_url = download_helper.GetDownloadURL( - self._PROJECT_NAME, self._PROJECT_VERSION) + self._PROJECT_NAME, project_version) expected_download_url = ( 'https://github.com/{0:s}/{1:s}/releases/download/{2:s}/' '{1:s}-{2:s}.tar.gz').format( self._PROJECT_ORGANIZATION, self._PROJECT_NAME, - self._PROJECT_VERSION) + project_version) self.assertEqual(download_url, expected_download_url)