From 9036f08e8c317c0b95459e8f4ab4e63bb8879282 Mon Sep 17 00:00:00 2001 From: Jose Diaz Date: Mon, 12 Jun 2023 06:02:25 +0200 Subject: [PATCH 1/3] Update download url from '2' to current '3', when navigating through table there is no even/odd classes in current ui, fixed issue with folder structures created where not the same referenced when files where saved --- script.py | 70 +++++++++++++++++--------------------------------- src/helpers.py | 14 +++++----- 2 files changed, 31 insertions(+), 53 deletions(-) diff --git a/script.py b/script.py index 36c6ffa..a8c98aa 100644 --- a/script.py +++ b/script.py @@ -45,31 +45,19 @@ def get_section_of_roms(section: str) -> List[models.ROM]: try: page: Response = requests.get('https://vimm.net/vault/' + section) soup = BeautifulSoup(page.content, 'html.parser') - result = soup.find( - 'table', {'class': 'rounded centered cellpadding1 hovertable'}) - for j in result.contents: + table = soup.select_one('table.rounded.centered.cellpadding1.hovertable') + result = table.select('tr > td[style="width:auto"] > a[href*="/vault/"]') + for j in result: if j != '\n': new_soup = BeautifulSoup(str(j), 'html.parser') - odd = new_soup.find(attrs={'class': 'odd'}) - even = new_soup.find(attrs={'class': 'even'}) - if odd is not None: - result_soup = BeautifulSoup(str(odd.contents[0]), - 'html.parser') - result = result_soup.find('a', href=True) - name = result.contents[0] - result = result['href'] - rom = models.ROM(name, result) - roms.append(rom) - odd = None - if even is not None: - result_soup = BeautifulSoup(str(even.contents[0]), - 'html.parser') - result = result_soup.find('a', href=True) - name = result.contents[0] - result = result['href'] - rom = models.ROM(name, result) - roms.append(rom) - even = None + result_soup: BeautifulSoup = BeautifulSoup( + str(new_soup), 'html.parser' + ) + result = result_soup.select_one('a') + name = result.text + result = result['href'] + rom = models.ROM(name, result) + roms.append(rom) except: e = sys.exc_info()[0] return roms @@ -127,7 +115,7 @@ def download_file(page_url: str, download_url: str, path: str) -> str: f'https://vimm.net/vault{page_url}' } file: Response = requests.get( - f'https://download2.vimm.net/download/?mediaId={download_url}', + f'https://download3.vimm.net/download/?mediaId={download_url}', headers=headers, allow_redirects=True) if file.status_code == 200: @@ -183,31 +171,19 @@ def get_system_search_section( try: page = requests.get(helpers.get_search_url(search_selection)) soup: BeautifulSoup = BeautifulSoup(page.content, 'html.parser') - result = soup.find( - 'table', {'class': 'rounded centered cellpadding1 hovertable'}) - for j in result.contents: + table = soup.select_one('table.rounded.centered.cellpadding1.hovertable') + result = table.select('tr > td[style="width:auto"] > a[href*="/vault/"]') + for j in result: if j != '\n': new_soup: BeautifulSoup = BeautifulSoup(str(j), 'html.parser') - odd = new_soup.find(attrs={'class': 'odd'}) - even = new_soup.find(attrs={'class': 'even'}) - if odd is not None: - result_soup: BeautifulSoup = BeautifulSoup( - str(odd.contents[0]), 'html.parser') - result = result_soup.find('a', href=True) - name = result.contents[0] - result = result['href'] - rom = models.ROM(name, result) - roms.append(rom) - odd = None - if even is not None: - result_soup = BeautifulSoup(str(even.contents[0]), - 'html.parser') - result = result_soup.find('a', href=True) - name = result.contents[0] - result = result['href'] - rom = models.ROM(name, result) - roms.append(rom) - even = None + result_soup: BeautifulSoup = BeautifulSoup( + str(new_soup), 'html.parser' + ) + result = result_soup.select_one('a') + name = result.text + result = result['href'] + rom = models.ROM(name, result) + roms.append(rom) except BaseException: e = sys.exc_info()[0] print('Failed on system search section') diff --git a/src/helpers.py b/src/helpers.py index f1310c0..1bbc833 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -174,20 +174,22 @@ def __create_all_w_home(path: str): def __create_sel_w_home(path: str, user_selections: List[str]): """Used in bulk mode when the user only wants selected systems if home dir is already created""" for x in user_selections: + system_path = __to_uri[__selections[int(x)][int(x)]] if not __check_if_system_dir_created(path, - __selections[int(x)][int(x)]): - __create_rom_system_dir(path, __selections[int(x)][int(x)]) - __create_alpha_num_structure(path, __selections[int(x)][int(x)]) + system_path): + __create_rom_system_dir(path, system_path) + __create_alpha_num_structure(path, system_path) def __create_sel_no_home(path: str, user_selections: List[str]): """Used in bulk mode when the user only wants selected systems""" __create_rom_home_dir(path) for x in user_selections: + system_path = __to_uri[__selections[int(x)][int(x)]] if not __check_if_system_dir_created(path, - __selections[int(x)][int(x)]): - __create_rom_system_dir(path, __selections[int(x)][int(x)]) - __create_alpha_num_structure(path, __selections[int(x)][int(x)]) + system_path): + __create_rom_system_dir(path, system_path) + __create_alpha_num_structure(path, system_path) def create_directory_structure(config: models.Config, path: str): From f91d5ff138f808c02de763d38ae9fad6708e237d Mon Sep 17 00:00:00 2001 From: Jose Diaz Date: Wed, 14 Jun 2023 10:20:26 +0200 Subject: [PATCH 2/3] Pick from download servers available a random --- script.py | 2 +- src/helpers.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/script.py b/script.py index a8c98aa..91dcbc6 100644 --- a/script.py +++ b/script.py @@ -115,7 +115,7 @@ def download_file(page_url: str, download_url: str, path: str) -> str: f'https://vimm.net/vault{page_url}' } file: Response = requests.get( - f'https://download3.vimm.net/download/?mediaId={download_url}', + f'{helpers.get_download_url_with_random_server_number()}/download/?mediaId={download_url}', headers=headers, allow_redirects=True) if file.status_code == 200: diff --git a/src/helpers.py b/src/helpers.py index 1bbc833..0a8fa05 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -231,6 +231,11 @@ def get_random_ua() -> str: return __user_agents[index] +def get_download_url_with_random_server_number() -> str: + """Returns a random download server number from available""" + download_servers_available = [2,3] + return f'https://download{random.choice(download_servers_available)}.vimm.net' + def print_welcome(): """Prints the welcome message..\ hmm yes the floor is made of floor""" From 3c91693fab08413cdb95649a9aef9552d5aa9784 Mon Sep 17 00:00:00 2001 From: Jose Diaz Date: Wed, 14 Jun 2023 10:23:53 +0200 Subject: [PATCH 3/3] Added new line after get_download_url_with_random_server_number --- src/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/helpers.py b/src/helpers.py index 0a8fa05..fea8db3 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -236,6 +236,7 @@ def get_download_url_with_random_server_number() -> str: download_servers_available = [2,3] return f'https://download{random.choice(download_servers_available)}.vimm.net' + def print_welcome(): """Prints the welcome message..\ hmm yes the floor is made of floor"""