Checks for erroneous data; new data types; reduced the number of photos needed to trigger a shorter timespan

CHANGELOG.md

@@ -1,3 +1,10 @@
+- **0.3.0** (2025-04-15):
+    - now collecting more data: tags, license, geo-accuracy
+    - re-downloading these data for existing records
+    - database schema versioning
+    - moved to PyPi TrustedPublisher auth
+    - migrated to pyproject.toml-only style
+
 - **0.2.1** (2024-01-08):
     - migrate to Digital Geography Lab’s GitHub
 

src/flickrhistory/__init__.py

@@ -15,4 +15,4 @@
 except ImportError:
     pass
 
-__version__ = "0.2.1"
+__version__ = "0.3.0.dev0"

src/flickrhistory/basicflickrhistorydownloader.py

@@ -21,7 +21,9 @@
 from .cache import Cache
 from .cacheupdaterthread import CacheUpdaterThread
 from .config import Config
+from .licensedownloader import LicenseDownloader
 from .photodownloaderthread import PhotoDownloaderThread
+from .photoupdaterthread import PhotoUpdaterThread
 from .sigtermreceivedexception import SigTermReceivedException
 from .timespan import TimeSpan
 from .userprofileupdaterthread import UserProfileUpdaterThread
@@ -30,7 +32,7 @@
 class BasicFlickrHistoryDownloader:
     """Download (all) georeferenced flickr posts."""
 
-    NUM_WORKERS = multiprocessing.cpu_count() + 1  # 1 == user_profile_updater
+    NUM_WORKERS = multiprocessing.cpu_count()
     NUM_MANAGERS = 2  # main thread + cache_updater
 
     # if output into pipe (e.g. logger, systemd), then
@@ -54,6 +56,8 @@ def __init__(self):
 
     def download(self):
         """Download all georeferenced flickr posts."""
+        LicenseDownloader(self._api_key_manager).update_licenses()
+
         for gap in self.gaps_in_download_history:
             self._todo_deque.append(gap)
 
@@ -74,6 +78,14 @@ def download(self):
                 worker.start()
                 self._worker_threads.append(worker)
 
+            # start photo record updaters
+            for i in range(self.NUM_WORKERS):
+                worker = PhotoUpdaterThread(
+                    self._api_key_manager, (i + 1, self.NUM_WORKERS)
+                )
+                worker.start()
+                self._worker_threads.append(worker)
+
             # start cache updater
             self._cache_updater_thread = CacheUpdaterThread(self._done_queue)
             self._cache_updater_thread.start()

src/flickrhistory/cacheupdaterthread.py

@@ -35,7 +35,10 @@ def run(self):
             try:
                 newly_downloaded = self._done_queue.get(timeout=0.1)
                 with Cache() as cache:
-                    cache["already downloaded"] += newly_downloaded
+                    try:
+                        cache["already downloaded"] += newly_downloaded
+                    except KeyError:
+                        cache["already downloaded"] = newly_downloaded
                     self.status = f"added {newly_downloaded}"
             except queue.Empty:
                 if self.shutdown.is_set():

src/flickrhistory/database/__init__.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+
+"""Database-related classes."""
+
+
+__all__ = [
+    "License",
+    "Photo",
+    "PhotoSaver",
+    "Session",
+    "User",
+    "UserSaver",
+]
+
+from .engine import Session
+from .models import License, Photo, User
+from .photo_saver import PhotoSaver
+from .user_saver import UserSaver

src/flickrhistory/database/databaseschemaupdater.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+"""Update the database schema if necessary."""
+
+
+__all__ = ["DatabaseSchemaUpdater"]
+
+
+import sys
+
+import sqlalchemy
+
+from .engine import engine
+
+
+# for now, schema updates are SQL only and work on PostgreSQL, only.
+# GeoAlchemy2 doesn’t really support SQLite, anyway
+SCHEMA_UPDATES = {
+    # 0 -> 1
+    1: """
+        ALTER TABLE
+            photos
+        ADD COLUMN IF NOT EXISTS
+            geo_accuracy SMALLINT;
+
+        CREATE TABLE IF NOT EXISTS
+            licenses (
+                id INTEGER,
+                name TEXT,
+                url TEXT
+            );
+
+        ALTER TABLE
+            photos
+        ADD COLUMN IF NOT EXISTS
+            license INTEGER REFERENCES licenses(id);
+    """,
+}
+
+
+class DatabaseSchemaUpdater:
+    """Update the database schema if necessary."""
+
+    LATEST = "LATEST"  # ‘magic’, see def set_schema_version
+
+    def __init__(self):
+        """Update the database schema if necessary."""
+        # Try to create database table for schema version
+        with engine.begin() as connection:
+            connection.execute(
+                sqlalchemy.text(
+                    """
+                        CREATE TABLE IF NOT EXISTS
+                            schema_versions
+                            (
+                                update TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+                                version INTEGER PRIMARY KEY
+                            );
+                    """
+                )
+            )
+
+    @property
+    def installed_version(self):
+        """Return current version."""
+        with engine.connect() as connection:
+            installed_version = connection.execute(
+                sqlalchemy.text(
+                    """
+                        SELECT
+                            COALESCE(
+                                MAX(version),
+                                0
+                            ) AS version
+                        FROM
+                            schema_versions;
+                    """
+                )
+            ).scalar_one_or_none()
+        return installed_version
+
+    def update_to_latest(self):
+        """Update to the latest schema version."""
+        installed_version = self.installed_version
+        while installed_version < max(SCHEMA_UPDATES.keys()):
+            print(
+                "Updating database schema (db version {:d}->{:d})".format(
+                    installed_version, installed_version + 1
+                ),
+                file=sys.stderr,
+                flush=True,  # so that we don’t seem without work
+            )
+            with engine.begin() as connection:
+                next_version = self.installed_version + 1
+                connection.execute(sqlalchemy.text(SCHEMA_UPDATES[next_version]))
+                self.set_schema_version(next_version)
+            installed_version = self.installed_version
+
+    @classmethod
+    def set_schema_version(cls, version):
+        """Set the schema version (without running update scripts)."""
+        if version == cls.LATEST:
+            version = max(SCHEMA_UPDATES.keys())
+        with engine.begin() as connection:
+            connection.execute(
+                sqlalchemy.text(
+                    """
+                    INSERT INTO
+                        schema_versions (version)
+                    VALUES (
+                        :version
+                    );
+                """
+                ),
+                {"version": version},
+            )

src/flickrhistory/database/engine.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+"""An SQLAlchemy engine and sessionmaker."""
+
+
+__all__ = ["engine", "Session"]
+
+
+import multiprocessing
+
+import sqlalchemy
+import sqlalchemy.orm
+
+from ..config import Config
+
+
+POOL_SIZE = multiprocessing.cpu_count() * 10
+
+
+with Config() as config:
+    engine = sqlalchemy.create_engine(
+        config["database_connection_string"],
+        pool_size=POOL_SIZE,
+        max_overflow=POOL_SIZE,
+    )
+
+
+if engine.dialect.name == "postgresql":
+    with engine.connect() as connection:
+        connection.execute(
+            sqlalchemy.text(
+                """
+                CREATE EXTENSION IF NOT EXISTS
+                    postgis;
+                """
+            )
+        )
+
+
+Session = sqlalchemy.orm.sessionmaker(engine, autoflush=False)

src/flickrhistory/database/models/__init__.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+"""ORM models for flickr entities."""
+
+
+__all__ = [
+    "License",
+    "Photo",
+    "User",
+    "Tag",
+]
+
+
+import sqlalchemy
+
+from ..engine import engine
+from .base import Base
+from ..databaseschemaupdater import DatabaseSchemaUpdater
+from .license import License
+from .photo import Photo
+from .tag import Tag
+from .user import User
+
+
+if sqlalchemy.inspect(engine).has_table(Photo.__table__.name):  # data exist
+    DatabaseSchemaUpdater().update_to_latest()
+else:
+    Base.metadata.create_all(engine)
+    DatabaseSchemaUpdater().set_schema_version(DatabaseSchemaUpdater.LATEST)

src/flickrhistory/database/models/base.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+"""A common sqlalchemy declarative_base() to share between models."""
+
+
+__all__ = ["Base"]
+
+
+import json
+import re
+
+import sqlalchemy.ext.declarative
+import sqlalchemy.orm
+
+
+CAMEL_CASE_TO_SNAKE_CASE_RE = re.compile(
+    "((?<=[a-z0-9])[A-Z]|(?!^)(?<!_)[A-Z](?=[a-z]))"
+)
+
+
+def camel_case_to_snake_case(camel_case):
+    """Convert a `camelCase` string to `snake_case`."""
+    snake_case = CAMEL_CASE_TO_SNAKE_CASE_RE.sub(r"_\1", camel_case).lower()
+    return snake_case
+
+
+class Base:
+    """Template for sqlalchemy declarative_base() to add shared functionality."""
+
+    def __str__(self):
+        """Return a str representation."""
+        primary_keys = {}
+        for pk in self.__mapper__.primary_key:
+            try:
+                primary_keys[pk.name] = getattr(self, pk.name)
+            except AttributeError:  # (not yet set)
+                pass
+        return f"<{self.__class__.__name__}({json.dumps(primary_keys)})>"
+
+    @sqlalchemy.orm.declared_attr
+    def __tablename__(cls):
+        """Return a table name derived from the class name."""
+        snake_case = camel_case_to_snake_case(cls.__name__)
+        return f"{snake_case}s"
+
+    def update(self, **kwargs):
+        """Update the values of this ORM object from keyword arguments."""
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+
+Base = sqlalchemy.ext.declarative.declarative_base(cls=Base)

src/flickrhistory/database/models/license.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+"""ORM class to represent a flickr license."""
+
+
+__all__ = ["License"]
+
+
+import sqlalchemy
+import sqlalchemy.orm
+
+from .base import Base
+
+
+class License(Base):
+    """ORM class to represent a flickr license."""
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    name = sqlalchemy.Column(sqlalchemy.Text)
+    url = sqlalchemy.Column(sqlalchemy.Text)
+    photos = sqlalchemy.orm.relationship("Photo")