From 120139fe56f1a688d8753d3e9711cdf391df8481 Mon Sep 17 00:00:00 2001 From: Jonas Mueller Date: Wed, 10 Apr 2024 08:27:10 +0200 Subject: [PATCH 1/3] Add Newline at the end of print --- BeFake/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BeFake/__main__.py b/BeFake/__main__.py index 59a169f..3a357b0 100644 --- a/BeFake/__main__.py +++ b/BeFake/__main__.py @@ -74,7 +74,7 @@ def refresh(bf): # Since this command is mostly used for debugging, it wouldn't be practical to add extra code to prevent this # behaviour. bf.firebase_refresh_tokens() - click.echo(bf.token, nl=False) + click.echo(bf.token) @cli.command(help="Download a feed") From e43a70b8fc488767efc36793fc4d7b208cfa07d8 Mon Sep 17 00:00:00 2001 From: Jonas Mueller Date: Wed, 10 Apr 2024 08:30:43 +0200 Subject: [PATCH 2/3] Add default save location --- BeFake/__main__.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/BeFake/__main__.py b/BeFake/__main__.py index 3a357b0..49991a1 100644 --- a/BeFake/__main__.py +++ b/BeFake/__main__.py @@ -21,7 +21,7 @@ logging.getLogger('urllib3').setLevel(logging.WARNING) BASE_DIR = Path.cwd() # TODO: make this configurable - +DATA_DIR = "data" def load_bf(func): """ @@ -79,13 +79,35 @@ def refresh(bf): @cli.command(help="Download a feed") @click.argument("feed_id", type=click.Choice(["friends", "friends-v1", "friends-of-friends", "discovery", "memories", "memories-v1"])) -@click.option("--save-location", required=True, help="Template for the paths where the posts should be downloaded") +@click.option("--save-location", help="Template for the paths where the posts should be downloaded") @click.option("--realmoji-location", help="Template for the paths where the (non-instant) realmojis should be downloaded") @click.option("--instant-realmoji-location", help="Template for the paths where the instant realmojis should be downloaded") @load_bf def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location): date_format = 'YYYY-MM-DD_HH-mm-ss' logging.debug(f"base dir: {BASE_DIR.absolute()}") + + if save_location is None: + if feed_id == "memories": + save_location = f"{DATA_DIR}" + "/feeds/memories/{date}" + elif feed_id == "memories-v1": + save_location = f"{DATA_DIR}" + "/feeds/memories-v1/{date}/{post_id}" + elif feed_id == "friends-v1": + save_location = f"{DATA_DIR}" + "/feeds/{feed_id}/{date}/{user}/{notification_id}/{post_id}" + else: + save_location = f"{DATA_DIR}" + "/feeds/{feed_id}/{date}/{user}/{post_id}" + + if realmoji_location is None: + if feed_id == "friends-v1": + realmoji_location = \ + f"{DATA_DIR}" + \ + "/feeds/{feed_id}/{post_date}/{post_user}/{notification_id}/{post_id}/reactions/{type}/{user}" + else: + realmoji_location = \ + f"{DATA_DIR}" + \ + "/feeds/{feed_id}/{post_date}/{post_user}/{post_id}/reactions/{type}/{user}" + + instant_realmoji_location = realmoji_location if instant_realmoji_location is None else instant_realmoji_location FEEDGETTER_MAP = { 'friends-v1': bf.get_friendsv1_feed, From e7f24e531aecc91757b86fa2904add282aa23601 Mon Sep 17 00:00:00 2001 From: Jonas Mueller Date: Wed, 10 Apr 2024 08:49:46 +0200 Subject: [PATCH 3/3] Add download-existing option --- BeFake/__main__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/BeFake/__main__.py b/BeFake/__main__.py index 49991a1..54eee2e 100644 --- a/BeFake/__main__.py +++ b/BeFake/__main__.py @@ -82,9 +82,10 @@ def refresh(bf): @click.option("--save-location", help="Template for the paths where the posts should be downloaded") @click.option("--realmoji-location", help="Template for the paths where the (non-instant) realmojis should be downloaded") @click.option("--instant-realmoji-location", help="Template for the paths where the instant realmojis should be downloaded") +@click.option("--download-existing", is_flag=True, help="Skip downloading posts that already exist") @load_bf -def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location): - date_format = 'YYYY-MM-DD_HH-mm-ss' +def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_location, download_existing): + date_format = 'YYYY-MM-DD' logging.debug(f"base dir: {BASE_DIR.absolute()}") if save_location is None: @@ -108,6 +109,12 @@ def feed(bf, feed_id, save_location, realmoji_location, instant_realmoji_locatio "/feeds/{feed_id}/{post_date}/{post_user}/{post_id}/reactions/{type}/{user}" instant_realmoji_location = realmoji_location if instant_realmoji_location is None else instant_realmoji_location + + download_existing = download_existing or False + + logging.debug(f"save location: {save_location}") + logging.debug(f"realmoji location: {realmoji_location}") + logging.debug(f"instant realmoji location: {instant_realmoji_location}") FEEDGETTER_MAP = { 'friends-v1': bf.get_friendsv1_feed, @@ -127,11 +134,11 @@ def _save_post_common(item, _save_location: Path): _save_location.mkdir(parents=True, exist_ok=True) (_save_location / "info.json").write_text(json.dumps(item.data_dict, indent=4)) - item.primary_photo.download(_save_location / "primary") - item.secondary_photo.download(_save_location / "secondary") + item.primary_photo.download(_save_location / "primary", skip_existing=not download_existing) + item.secondary_photo.download(_save_location / "secondary", skip_existing=not download_existing) if item.bts_video.exists(): # FIXME: bts_video successfully instantiates when there is none, but download() would fail - item.bts_video.download(_save_location / "bts") + item.bts_video.download(_save_location / "bts", skip_existing=not download_existing) def _save_realmojis(post, realmoji_location: str, instant_realmoji_location: str): for emoji in post.realmojis: