From 216b28d35e680cc2bd9ff074ccc42938900a7a22 Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 13:01:54 -0300 Subject: [PATCH 1/9] feature: it works, is ugly --- shbin.py | 57 ++++++++++++++++++++++++++++++++++++++--------- tests/test_cli.py | 15 ++++++++++++- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/shbin.py b/shbin.py index a2967f5..47cbea6 100644 --- a/shbin.py +++ b/shbin.py @@ -7,6 +7,7 @@ import re import secrets import sys +import requests from mimetypes import guess_extension import pyclip @@ -59,6 +60,23 @@ def get_repo_and_user(): gh = Github(os.environ["SHBIN_GITHUB_TOKEN"]) return gh.get_repo(os.environ["SHBIN_REPO"]), gh.get_user().login +def get_repo(full_name_repo): + gh = Github(os.environ["SHBIN_GITHUB_TOKEN"]) + return gh.get_repo(full_name_repo) + +def get_path(url): + pattern = r"https://github.com/.*?/(?:tree|blob)/(.*)" + result = re.findall(pattern, url) + if result: + extracted_string = result[0] + return extracted_string + else: + raise DocoptExit( + f"Ensure your path is from github repository. (error {e})" + ) + + + def expand_paths(path_or_patterns): """ @@ -99,19 +117,38 @@ def download(url_or_path, repo, user): $ shbin dl https://github.com/Shiphero/pastebin/blob/main/bibo/AWS_API_fullfilment_methods/ $ shbin dl bibo/AWS_API_fullfilment_methods/ """ - path = re.sub(rf"^https://github\.com/{repo.full_name}/(blob|tree)/{repo.default_branch}/", "", url_or_path) + full_name_repo = repo.full_name + pattern = r"https://github.com/(.*?)/(tree|blob)/" + result = re.findall(pattern, url_or_path) + if result: + full_name_repo = result[0][0] + + is_from_my_repo = True if "https://" not in url_or_path or full_name_repo==repo.full_name else False + if is_from_my_repo: + path = re.sub(rf"^https://github\.com/{repo.full_name}/(blob|tree)/{repo.default_branch}/", "", url_or_path) + else: + repo = get_repo(full_name_repo) + path = get_path(url_or_path) + path = path.rstrip("/") try: - content = repo.get_contents(path) - if isinstance(content, list): - # FIXME currently this will flatten the tree: - # suposse dir/foo.py and dir/subdir/bar.py - # Then `$ shbin dl dir` will get foo.py and bar.py in the same dir. - for content_file in content: - download(content_file.path, repo, user) - return + if is_from_my_repo: + content = repo.get_contents(path) + if isinstance(content, list): + # FIXME currently this will flatten the tree: + # suposse dir/foo.py and dir/subdir/bar.py + # Then `$ shbin dl dir` will get foo.py and bar.py in the same dir. + for content_file in content: + download(content_file.path, repo, user) + return + else: + content = content.decoded_content else: - content = content.decoded_content + url = f"https://raw.githubusercontent.com/{repo.full_name}/{path}" + response = requests.get(url) + if response.status_code > 200: + raise GithubException + content = response.content except GithubException: print("[red]x[/red] content not found") else: diff --git a/tests/test_cli.py b/tests/test_cli.py index dc5fcb6..6abf984 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -32,6 +32,7 @@ def repo(): repo = create_autospec(Repository, name="gh_repo") repo.create_file.return_value = {"content": Mock(html_url="https://the-url")} repo.update_file.return_value = {"content": Mock(html_url="https://the-url-updated")} + repo.full_name = 'my_awesome/repository' return repo @@ -320,7 +321,19 @@ def test_force_new(pyclip, tmp_path, patched_repo_and_user, repo, capsys): assert capsys.readouterr().out == "🔗📋 https://the-url-2\n" -def test_download_a_file(tmp_path, patched_repo_and_user, repo): +def test_download_a_file_from_owned_repo(tmp_path, patched_repo_and_user, repo): + import ipdb;ipdb.set_trace() + git_data = { + "decoded_content": b"awesome content", + } + repo.get_contents.return_value = create_github_downloable_files(git_data) + working_dir = tmp_path / "working_dir" + working_dir.mkdir() + os.chdir(working_dir) + main(["dl", "hello.md"]) + assert (working_dir / "hello.md").read_bytes() == b"awesome content" + +def test_download_a_file_from_public_repo(tmp_path, patched_repo_and_user, repo): git_data = { "decoded_content": b"awesome content", } From 66675c78748a9d06748224b9fef1de0b6c635f7a Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 15:28:41 -0300 Subject: [PATCH 2/9] feature: added tests --- shbin.py | 5 ++++- tests/test_cli.py | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/shbin.py b/shbin.py index 47cbea6..d9a56be 100644 --- a/shbin.py +++ b/shbin.py @@ -123,6 +123,7 @@ def download(url_or_path, repo, user): if result: full_name_repo = result[0][0] + is_from_my_repo = True if "https://" not in url_or_path or full_name_repo==repo.full_name else False if is_from_my_repo: path = re.sub(rf"^https://github\.com/{repo.full_name}/(blob|tree)/{repo.default_branch}/", "", url_or_path) @@ -147,10 +148,12 @@ def download(url_or_path, repo, user): url = f"https://raw.githubusercontent.com/{repo.full_name}/{path}" response = requests.get(url) if response.status_code > 200: - raise GithubException + raise Exception("There was a problem with your download, please check url") content = response.content except GithubException: print("[red]x[/red] content not found") + except Exception as e: + print(f"[red]x[/red] {e}") else: target = pathlib.Path(path).name pathlib.Path(target).write_bytes(content) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6abf984..4ff7539 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -36,6 +36,13 @@ def repo(): return repo +@pytest.fixture +def outside_repo(): + outside_repo = create_autospec(Repository, name="gh_repo") + outside_repo.full_name = 'another_awesome/repository' + return outside_repo + + @pytest.fixture(autouse=True) def pyclip(monkeypatch): class Stub: @@ -71,6 +78,12 @@ def patched_repo_and_user(repo): yield mocked +@pytest.fixture +def patched_another_repo_and_user(outside_repo): + with patch("shbin.get_repo", return_value=(outside_repo)) as mocked: + yield mocked + + @pytest.mark.parametrize("argv", (["-h"], ["--help"])) def test_help(capsys, argv): with pytest.raises(SystemExit): @@ -322,7 +335,6 @@ def test_force_new(pyclip, tmp_path, patched_repo_and_user, repo, capsys): def test_download_a_file_from_owned_repo(tmp_path, patched_repo_and_user, repo): - import ipdb;ipdb.set_trace() git_data = { "decoded_content": b"awesome content", } @@ -333,13 +345,25 @@ def test_download_a_file_from_owned_repo(tmp_path, patched_repo_and_user, repo): main(["dl", "hello.md"]) assert (working_dir / "hello.md").read_bytes() == b"awesome content" -def test_download_a_file_from_public_repo(tmp_path, patched_repo_and_user, repo): - git_data = { - "decoded_content": b"awesome content", - } - repo.get_contents.return_value = create_github_downloable_files(git_data) + +def test_download_a_file_from_public_repo(tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, + patched_repo_and_user): + requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', + content=b'awesome content') working_dir = tmp_path / "working_dir" working_dir.mkdir() os.chdir(working_dir) - main(["dl", "hello.md"]) + main(["dl", "https://github.com/another_awesome/repository/blob/main/hello.md"]) assert (working_dir / "hello.md").read_bytes() == b"awesome content" + +def test_download_a_file_from_public_repo_rasie_error(tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, + patched_repo_and_user): + requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', + status_code=400) + working_dir = tmp_path / "working_dir" + working_dir.mkdir() + os.chdir(working_dir) + main(["dl", "https://github.com/another_awesome/repository/blob/main/hello.md"]) + with pytest.raises(Exception) as exc_info: + raise Exception('There was a problem with your download, please check url') + assert str(exc_info.value) == 'There was a problem with your download, please check url' \ No newline at end of file From 0d8810b8f1147a4946f1ce9403d5c41881791baf Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 15:37:14 -0300 Subject: [PATCH 3/9] feature: added exception for non gh pages --- shbin.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shbin.py b/shbin.py index d9a56be..7841ed6 100644 --- a/shbin.py +++ b/shbin.py @@ -119,9 +119,15 @@ def download(url_or_path, repo, user): """ full_name_repo = repo.full_name pattern = r"https://github.com/(.*?)/(tree|blob)/" - result = re.findall(pattern, url_or_path) - if result: - full_name_repo = result[0][0] + try: + result = re.findall(pattern, url_or_path) + if result: + full_name_repo = result[0][0] + else: + raise ValueError(f"The URL is not from a public repository on GitHub") + except ValueError as ve: + print(f"[red]x[/red] {ve}") + return is_from_my_repo = True if "https://" not in url_or_path or full_name_repo==repo.full_name else False From 5a0dab59f3c43078efe86edf738650c45b4c96c9 Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 16:00:21 -0300 Subject: [PATCH 4/9] feature: remove exception for non gh pages --- shbin.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/shbin.py b/shbin.py index 7841ed6..608c0c4 100644 --- a/shbin.py +++ b/shbin.py @@ -119,18 +119,11 @@ def download(url_or_path, repo, user): """ full_name_repo = repo.full_name pattern = r"https://github.com/(.*?)/(tree|blob)/" - try: - result = re.findall(pattern, url_or_path) - if result: - full_name_repo = result[0][0] - else: - raise ValueError(f"The URL is not from a public repository on GitHub") - except ValueError as ve: - print(f"[red]x[/red] {ve}") - return - + result = re.findall(pattern, url_or_path) + if result: + full_name_repo = result[0][0] - is_from_my_repo = True if "https://" not in url_or_path or full_name_repo==repo.full_name else False + is_from_my_repo = "https://" not in url_or_path or full_name_repo == repo.full_name if is_from_my_repo: path = re.sub(rf"^https://github\.com/{repo.full_name}/(blob|tree)/{repo.default_branch}/", "", url_or_path) else: From 597ed7158607fe9e2d5a6d2dd093a68d17522e66 Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 16:04:17 -0300 Subject: [PATCH 5/9] feature: add requests_mock to dev env --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e64aa0e..875843e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dev = [ "black", "pytest-cov", "flake8", + "requests_mock" ] [tool.black] From ec2db69fbb2e3142d43d68ef833704614618f07c Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 16:06:16 -0300 Subject: [PATCH 6/9] fix: black formatter --- tests/test_cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4ff7539..04aecf7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -356,8 +356,10 @@ def test_download_a_file_from_public_repo(tmp_path, patched_another_repo_and_use main(["dl", "https://github.com/another_awesome/repository/blob/main/hello.md"]) assert (working_dir / "hello.md").read_bytes() == b"awesome content" -def test_download_a_file_from_public_repo_rasie_error(tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, - patched_repo_and_user): + +def test_download_a_file_from_public_repo_rasie_error(tmp_path, patched_another_repo_and_user, outside_repo, + requests_mock, + patched_repo_and_user): requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', status_code=400) working_dir = tmp_path / "working_dir" @@ -366,4 +368,5 @@ def test_download_a_file_from_public_repo_rasie_error(tmp_path, patched_another_ main(["dl", "https://github.com/another_awesome/repository/blob/main/hello.md"]) with pytest.raises(Exception) as exc_info: raise Exception('There was a problem with your download, please check url') - assert str(exc_info.value) == 'There was a problem with your download, please check url' \ No newline at end of file + assert str(exc_info.value) == 'There was a problem with your download, please check url' + \ No newline at end of file From 69aa524fe3bf3fd4b38e71eff07c02bb7cb361a7 Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 16:12:50 -0300 Subject: [PATCH 7/9] fix: black formatter --- shbin.py | 4 +--- tests/test_cli.py | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/shbin.py b/shbin.py index 608c0c4..e6942ed 100644 --- a/shbin.py +++ b/shbin.py @@ -164,9 +164,7 @@ def main(argv=None) -> None: try: repo, user = get_repo_and_user() except Exception as e: - raise DocoptExit( - f"Ensure SHBIN_GITHUB_TOKEN and SHBIN_REPO environment variables are correctly set. (error {e})" - ) + raise DocoptExit(f"Ensure SHBIN_GITHUB_TOKEN and SHBIN_REPO environment variables are correctly set. (error {e})") # resolves namespace + target-dir (without ending slash) # it also interpolates {user} namespace = args.get("--namespace") diff --git a/tests/test_cli.py b/tests/test_cli.py index 04aecf7..e5cb5d4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -32,14 +32,14 @@ def repo(): repo = create_autospec(Repository, name="gh_repo") repo.create_file.return_value = {"content": Mock(html_url="https://the-url")} repo.update_file.return_value = {"content": Mock(html_url="https://the-url-updated")} - repo.full_name = 'my_awesome/repository' + repo.full_name = "my_awesome/repository" return repo @pytest.fixture def outside_repo(): outside_repo = create_autospec(Repository, name="gh_repo") - outside_repo.full_name = 'another_awesome/repository' + outside_repo.full_name = "another_awesome/repository" return outside_repo @@ -346,10 +346,12 @@ def test_download_a_file_from_owned_repo(tmp_path, patched_repo_and_user, repo): assert (working_dir / "hello.md").read_bytes() == b"awesome content" -def test_download_a_file_from_public_repo(tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, - patched_repo_and_user): - requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', - content=b'awesome content') +def test_download_a_file_from_public_repo( + tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, patched_repo_and_user +): + requests_mock.get( + 'https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', content=b"awesome content" + ) working_dir = tmp_path / "working_dir" working_dir.mkdir() os.chdir(working_dir) @@ -357,11 +359,10 @@ def test_download_a_file_from_public_repo(tmp_path, patched_another_repo_and_use assert (working_dir / "hello.md").read_bytes() == b"awesome content" -def test_download_a_file_from_public_repo_rasie_error(tmp_path, patched_another_repo_and_user, outside_repo, - requests_mock, - patched_repo_and_user): - requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', - status_code=400) +def test_download_a_file_from_public_repo_rasie_error( + tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, patched_repo_and_user +): + requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', status_code=400) working_dir = tmp_path / "working_dir" working_dir.mkdir() os.chdir(working_dir) @@ -369,4 +370,5 @@ def test_download_a_file_from_public_repo_rasie_error(tmp_path, patched_another_ with pytest.raises(Exception) as exc_info: raise Exception('There was a problem with your download, please check url') assert str(exc_info.value) == 'There was a problem with your download, please check url' - \ No newline at end of file + + From 3539c28dbff4f17460d53c61a59aa07b8654d8da Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 16:15:55 -0300 Subject: [PATCH 8/9] fix: black formatter, works better if I install black isn't? --- shbin.py | 12 ++++++------ tests/test_cli.py | 14 ++++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/shbin.py b/shbin.py index e6942ed..196ab11 100644 --- a/shbin.py +++ b/shbin.py @@ -60,10 +60,12 @@ def get_repo_and_user(): gh = Github(os.environ["SHBIN_GITHUB_TOKEN"]) return gh.get_repo(os.environ["SHBIN_REPO"]), gh.get_user().login + def get_repo(full_name_repo): gh = Github(os.environ["SHBIN_GITHUB_TOKEN"]) return gh.get_repo(full_name_repo) + def get_path(url): pattern = r"https://github.com/.*?/(?:tree|blob)/(.*)" result = re.findall(pattern, url) @@ -71,11 +73,7 @@ def get_path(url): extracted_string = result[0] return extracted_string else: - raise DocoptExit( - f"Ensure your path is from github repository. (error {e})" - ) - - + raise DocoptExit(f"Ensure your path is from github repository. (error {e})") def expand_paths(path_or_patterns): @@ -164,7 +162,9 @@ def main(argv=None) -> None: try: repo, user = get_repo_and_user() except Exception as e: - raise DocoptExit(f"Ensure SHBIN_GITHUB_TOKEN and SHBIN_REPO environment variables are correctly set. (error {e})") + raise DocoptExit( + f"Ensure SHBIN_GITHUB_TOKEN and SHBIN_REPO environment variables are correctly set. (error {e})" + ) # resolves namespace + target-dir (without ending slash) # it also interpolates {user} namespace = args.get("--namespace") diff --git a/tests/test_cli.py b/tests/test_cli.py index e5cb5d4..2149922 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -347,10 +347,10 @@ def test_download_a_file_from_owned_repo(tmp_path, patched_repo_and_user, repo): def test_download_a_file_from_public_repo( - tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, patched_repo_and_user + tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, patched_repo_and_user ): requests_mock.get( - 'https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', content=b"awesome content" + "https://raw.githubusercontent.com/another_awesome/repository/main/hello.md", content=b"awesome content" ) working_dir = tmp_path / "working_dir" working_dir.mkdir() @@ -360,15 +360,13 @@ def test_download_a_file_from_public_repo( def test_download_a_file_from_public_repo_rasie_error( - tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, patched_repo_and_user + tmp_path, patched_another_repo_and_user, outside_repo, requests_mock, patched_repo_and_user ): - requests_mock.get('https://raw.githubusercontent.com/another_awesome/repository/main/hello.md', status_code=400) + requests_mock.get("https://raw.githubusercontent.com/another_awesome/repository/main/hello.md", status_code=400) working_dir = tmp_path / "working_dir" working_dir.mkdir() os.chdir(working_dir) main(["dl", "https://github.com/another_awesome/repository/blob/main/hello.md"]) with pytest.raises(Exception) as exc_info: - raise Exception('There was a problem with your download, please check url') - assert str(exc_info.value) == 'There was a problem with your download, please check url' - - + raise Exception("There was a problem with your download, please check url") + assert str(exc_info.value) == "There was a problem with your download, please check url" From 0ca4c64d6791b3b0b5d829a0caf46c1fd152c352 Mon Sep 17 00:00:00 2001 From: Alvar Maciel Date: Fri, 16 Jun 2023 16:20:39 -0300 Subject: [PATCH 9/9] Update tests/test_cli.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Martín Gaitán --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2149922..ca40ffb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -80,7 +80,7 @@ def patched_repo_and_user(repo): @pytest.fixture def patched_another_repo_and_user(outside_repo): - with patch("shbin.get_repo", return_value=(outside_repo)) as mocked: + with patch("shbin.get_repo", return_value=outside_repo) as mocked: yield mocked