From 69ae2ea798616b89c7ab3a0b494628f8cf15eaef Mon Sep 17 00:00:00 2001 From: Jonas Mueller Date: Fri, 5 Apr 2024 14:54:47 +0200 Subject: [PATCH 1/2] Add retries with wait when downloading --- BeFake/models/picture.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/BeFake/models/picture.py b/BeFake/models/picture.py index 307334d..145c1e8 100644 --- a/BeFake/models/picture.py +++ b/BeFake/models/picture.py @@ -1,4 +1,5 @@ import datetime +import time import io import logging import os.path @@ -26,7 +27,7 @@ def __repr__(self) -> str: def exists(self): return self.url is not None - def download(self, path: Path | None, skip_existing: bool = True) -> bytes | None: + def download(self, path: Path, skip_existing: bool = True, retry_count: int=10, wait_time: int = 2) -> bytes: """ path: Path to save the image to (without extension). If None, the image is not saved. """ @@ -39,9 +40,27 @@ def download(self, path: Path | None, skip_existing: bool = True) -> bytes | Non logging.debug(f"Skipping already-downloaded {self.url}") return - r = httpx.get(self.url, headers={ - "user-agent": "BeReal/1.0.1 (AlexisBarreyat.BeReal; build:9513; iOS 16.0.2) 1.0.0/BRApriKit", - "x-ios-bundle-identifier": "AlexisBarreyat.BeReal"}) + got_data = False + count = 0 + while not got_data and count < retry_count: + try: + r = httpx.get(self.url, headers={ + "user-agent": "BeReal/1.0.1 (AlexisBarreyat.BeReal; build:9513; iOS 16.0.2) 1.0.0/BRApriKit", + "x-ios-bundle-identifier": "AlexisBarreyat.BeReal"}) + got_data = True + if r.status_code == 200: + got_data = True + else: + print(f"Error requesting image: {r.status_code}, retrying in 5 seconds") + time.sleep(wait_time) + except Exception as e: + count += 1 + logging.error(f"Error downloading {self.url}: {e}") + time.sleep(wait_time) + count += 1 + + if not got_data: + raise Exception(f"Error requesting image: {r.status_code} with {retry_count} retries!") self.data = r.content From 041cafaeedc6ecf3a2dbb5585dc1a562e0d50102 Mon Sep 17 00:00:00 2001 From: Jonas Mueller Date: Fri, 5 Apr 2024 14:55:17 +0200 Subject: [PATCH 2/2] Add days-back option to feed memories-v1 --- BeFake/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/BeFake/__main__.py b/BeFake/__main__.py index 59a169f..58163c6 100644 --- a/BeFake/__main__.py +++ b/BeFake/__main__.py @@ -4,6 +4,7 @@ import string from functools import wraps from pathlib import Path +from datetime import datetime import click from rich.logging import RichHandler @@ -82,8 +83,9 @@ def refresh(bf): @click.option("--save-location", required=True, help="Template for the paths where the posts should be downloaded") @click.option("--realmoji-location", help="Template for the paths where the (non-instant) realmojis should be downloaded") @click.option("--instant-realmoji-location", help="Template for the paths where the instant realmojis should be downloaded") +@click.option("--days-back", type=int, help="The number of days to go back in time to download memories, does not effect other feeds") @load_bf -def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location): +def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location, days_back): date_format = 'YYYY-MM-DD_HH-mm-ss' logging.debug(f"base dir: {BASE_DIR.absolute()}") @@ -142,6 +144,11 @@ def _save_realmojis(post, realmoji_location: str, instant_realmoji_location: str _save_post_common(item, _save_location) elif feed_id == "memories-v1": + if days_back: + memoryDay = datetime.strptime(item.memory_day, '%Y-%m-%d') + if (datetime.now() - memoryDay).days > days_back: + continue + logging.info(f"saving memory {item.memory_day}".ljust(50, " ") + item.id) _save_location = BASE_DIR / save_location.format(date=item.memory_day, post_id=item.id) _save_post_common(item, _save_location)