Skip to content
This repository was archived by the owner on Jan 30, 2025. It is now read-only.

Add retry and days-back option #5

Open
jjj120 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add retry and days-back option #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
69ae2ea
Add retries with wait when downloading
jjj120 Apr 5, 2024
041cafa
Add days-back option to feed memories-v1
jjj120 Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion BeFake/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import string
from functools import wraps
from pathlib import Path
from datetime import datetime

import click
from rich.logging import RichHandler
Expand Down Expand Up @@ -82,8 +83,9 @@ def refresh(bf):
@click.option("--save-location", required=True, help="Template for the paths where the posts should be downloaded")
@click.option("--realmoji-location", help="Template for the paths where the (non-instant) realmojis should be downloaded")
@click.option("--instant-realmoji-location", help="Template for the paths where the instant realmojis should be downloaded")
@click.option("--days-back", type=int, help="The number of days to go back in time to download memories, does not effect other feeds")
@load_bf
def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location):
def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location, days_back):
date_format = 'YYYY-MM-DD_HH-mm-ss'
logging.debug(f"base dir: {BASE_DIR.absolute()}")

Expand Down Expand Up @@ -142,6 +144,11 @@ def _save_realmojis(post, realmoji_location: str, instant_realmoji_location: str
_save_post_common(item, _save_location)

elif feed_id == "memories-v1":
if days_back:
memoryDay = datetime.strptime(item.memory_day, '%Y-%m-%d')
if (datetime.now() - memoryDay).days > days_back:
continue

logging.info(f"saving memory {item.memory_day}".ljust(50, " ") + item.id)
_save_location = BASE_DIR / save_location.format(date=item.memory_day, post_id=item.id)
_save_post_common(item, _save_location)
Expand Down
27 changes: 23 additions & 4 deletions BeFake/models/picture.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import datetime
import time
import io
import logging
import os.path
Expand Down Expand Up @@ -26,7 +27,7 @@ def __repr__(self) -> str:
def exists(self):
return self.url is not None

def download(self, path: Path | None, skip_existing: bool = True) -> bytes | None:
def download(self, path: Path, skip_existing: bool = True, retry_count: int=10, wait_time: int = 2) -> bytes:
"""
path: Path to save the image to (without extension). If None, the image is not saved.
"""
Expand All @@ -39,9 +40,27 @@ def download(self, path: Path | None, skip_existing: bool = True) -> bytes | Non
logging.debug(f"Skipping already-downloaded {self.url}")
return

r = httpx.get(self.url, headers={
"user-agent": "BeReal/1.0.1 (AlexisBarreyat.BeReal; build:9513; iOS 16.0.2) 1.0.0/BRApriKit",
"x-ios-bundle-identifier": "AlexisBarreyat.BeReal"})
got_data = False
count = 0
while not got_data and count < retry_count:
try:
r = httpx.get(self.url, headers={
"user-agent": "BeReal/1.0.1 (AlexisBarreyat.BeReal; build:9513; iOS 16.0.2) 1.0.0/BRApriKit",
"x-ios-bundle-identifier": "AlexisBarreyat.BeReal"})
got_data = True
if r.status_code == 200:
got_data = True
else:
print(f"Error requesting image: {r.status_code}, retrying in 5 seconds")
time.sleep(wait_time)
except Exception as e:
count += 1
logging.error(f"Error downloading {self.url}: {e}")
time.sleep(wait_time)
count += 1

if not got_data:
raise Exception(f"Error requesting image: {r.status_code} with {retry_count} retries!")

self.data = r.content

Expand Down