Anonymous credentials draft endpoints

fastpath/clickhouse_init.sql

@@ -38,7 +38,12 @@ CREATE TABLE IF NOT EXISTS default.fastpath
     `blocking_type` String,
     `test_helper_address` LowCardinality(String),
     `test_helper_type` LowCardinality(String),
-    `ooni_run_link_id` Nullable(UInt64)
+    `ooni_run_link_id` Nullable(UInt64),
+    `is_verified` Int8,
+    `nym` Nullable(String),
+    `zkp_request` Nullable(String),
+    `age_range` Nullable(String),
+    `msm_range` Nullable(String),
 )
 ENGINE = ReplacingMergeTree
 ORDER BY (measurement_start_time, report_id, input)
@@ -202,4 +207,3 @@ CREATE TABLE IF NOT EXISTS default.fingerprints_http
 )
 ENGINE = EmbeddedRocksDB
 PRIMARY KEY name;
-

fastpath/debian/changelog

@@ -1,3 +1,9 @@
+fastpath (0.89) unstable; urgency=medium
+
+  * Add support for new anonymous credentials database fields
+
+ -- Luis Diaz <luis@openobservatory.org> Tue, 15 Sep 2026 16:25:00 +0100
+
 fastpath (0.88) unstable; urgency=medium
 
   * Remove fingerprints bytes metrics

fastpath/fastpath/core.py

@@ -1616,7 +1616,7 @@ def flag_measurements_with_wrong_date(msm: dict, msmt_uid: str, scores: dict) ->
         scores["msg"] = "Measurement start time too old"
 
 def write_measurement_to_disk(msm_tup) -> None:
-    """Write this measurement to disk so that it can be 
+    """Write this measurement to disk so that it can be
     processed by the measurement uploader
 
     Args:
@@ -1633,7 +1633,7 @@ def write_measurement_to_disk(msm_tup) -> None:
     msmtdir = spooldir / "incoming" / dirname
     msmtdir.mkdir(parents=True, exist_ok=True)
 
-    try: 
+    try:
         msmt_f_tmp = msmtdir / f"{msmt_uid}.post.tmp"
         msmt_f_tmp.write_bytes(data)
         msmt_f = msmtdir / f"{msmt_uid}.post"
@@ -1654,7 +1654,14 @@ def process_measurement(msm_tup, buffer_writes=False) -> None:
         assert msmt_uid
         if measurement is None:
             measurement = ujson.loads(msm_jstr)
-        if sorted(measurement.keys()) == ["content", "format"]:
+
+        is_verified = g(measurement, 'is_verified', False)
+        nym = g(measurement, 'nym')
+        zkp_request = g(measurement, 'zkp_request')
+        age_range = g(measurement, 'age_range')
+        msm_range = g(measurement, 'msm_range')
+
+        if "content" in measurement and "format" in measurement:
             measurement = unwrap_msmt(measurement)
         rid = measurement.get("report_id")
         inp = measurement.get("input")
@@ -1742,6 +1749,11 @@ def process_measurement(msm_tup, buffer_writes=False) -> None:
             test_helper_address,
             test_helper_type,
             ooni_run_link_id,
+            is_verified,
+            nym,
+            zkp_request,
+            age_range,
+            msm_range,
             buffer_writes=buffer_writes,
         )
 

fastpath/fastpath/db.py

@@ -10,7 +10,7 @@
 from datetime import datetime
 from textwrap import dedent
 from urllib.parse import urlparse
-from typing import List, Tuple, Dict, Optional
+from typing import List, Tuple, Dict, Optional, Any
 import logging
 
 try:
@@ -209,6 +209,11 @@ def clickhouse_upsert_summary(
     test_helper_address: str,
     test_helper_type: str,
     ooni_run_link_id: Optional[int],
+    is_verified: bool,
+    zkp_request: Optional[str],
+    nym: Optional[str],
+    age_range: Optional[Tuple[int, int]],
+    msm_range: Optional[Tuple[int, int]],
     buffer_writes=False,
 ) -> None:
     """Insert a row in the fastpath table. Overwrite an existing one."""
@@ -224,6 +229,10 @@ def nn(features: dict, k: str) -> str:
     def tf(v: bool) -> str:
         return "t" if v else "f"
 
+    def serialize_optional(x : Optional[Any]) -> Optional[str]:
+        """Serialize to string if not None, return None otherwise"""
+        return ujson.dumps(x) if x is not None else None
+
     test_name = msm.get("test_name", None) or ""
     input_, domain = extract_input_domain(msm, test_name)
     asn = int(msm["probe_asn"][2:])  # AS123
@@ -257,6 +266,11 @@ def tf(v: bool) -> str:
         test_helper_address=test_helper_address,
         test_helper_type=test_helper_type,
         ooni_run_link_id=ooni_run_link_id,
+        is_verified=tf(is_verified),
+        nym=nym,
+        zkp_request=zkp_request,
+        age_range=serialize_optional(age_range),
+        msm_range=serialize_optional(msm_range)
     )
 
     if buffer_writes:

fastpath/fastpath/tests/test_functional_nodb.py

@@ -154,6 +154,11 @@ def test_score_web_connectivity_bug_610_2(fprints):
             "test_helper_address": "https://0.th.ooni.org",
             "test_helper_type": "https",
             "ooni_run_link_id": None,
+            "is_verified" : "f",
+            "nym" : None,
+            "zkp_request" : None,
+            "age_range" : None,
+            "msm_range" : None,
         }
     ]
 
@@ -201,6 +206,11 @@ def test_score_browser_web(fprints):
             "test_runtime": 0.35740000000037253,
             "test_start_time": datetime.datetime(2023, 3, 20, 18, 26, 35),
             "test_version": "0.1.0",
+            "is_verified" : "f",
+            "nym" : None,
+            "zkp_request" : None,
+            "age_range" : None,
+            "msm_range" : None,
         },
     ]
 
@@ -252,6 +262,11 @@ def test_score_openvpn():
             "test_helper_address": "",
             "test_helper_type": "",
             "ooni_run_link_id": None,
+            "is_verified" : "f",
+            "nym" : None,
+            "zkp_request" : None,
+            "age_range" : None,
+            "msm_range" : None,
         }
     ]
 

fastpath/makefile

@@ -80,18 +80,18 @@ docker:
 
 # Runs docker in foreground, useful for checking errors in the image before it runs
 docker-fg:
-	docker compose --profile default up --build 
+	docker compose --profile default up --build
 
-# Runs both fastpath and the testing clickhous. 
+# Runs both fastpath and the testing clickhous.
 # Mind the fastpath configuration in fastpath.conf
 docker-all: docker-clickhouse
 	echo "Waiting for clickhouse..."
-	sleep 4 
-	docker compose --profile default up --build -d 
+	sleep 4
+	docker compose --profile default up --build -d
 
 # Turns off every service
 docker-down:
-	docker compose --profile all down 
+	docker compose --profile all down
 
 # If you need to test the fastpath locally you can use this rule to spawn the clickhouse database
 # locally and then use `make docker` or `make docker-fg` to start the fastpath container. Ex:
@@ -109,15 +109,15 @@ docker-login:
 
 # Get logs from the fastpath docker service
 docker-logs:
-	docker compose logs fastpath -f 
+	docker compose logs fastpath -f
 
-# Get logs for a specified service. Example: 
+# Get logs for a specified service. Example:
 # `make docker-logs-for args="clickhouse-server"`
 docker-logs-for:
-	docker compose logs $(args) -f 
+	docker compose logs $(args) -f
 
 # Used for actually building the fastpath docker image
-docker-build: 
+docker-build:
 # We need to use tar -czh to resolve the common dir symlink
 	tar -czh . | docker build \
 		--build-arg BUILD_LABEL=${BUILD_LABEL} \

ooniapi/common/src/common/config.py

@@ -1,5 +1,6 @@
 from typing import List, Dict
 from pydantic_settings import BaseSettings
+from pydantic import Field
 
 
 class Settings(BaseSettings):
@@ -35,13 +36,27 @@ class Settings(BaseSettings):
 
     # Where the geoip DBs are downloaded to
     geoip_db_dir: str = "/var/lib/ooni/geoip"
-    # Ooniprobe only
+    # -- < Ooniprobe only > -------------------------------------------------------------
     msmt_spool_dir: str = ""
     fastpath_url: str = ""  # example: http://123.123.123.123:8472
     failed_reports_bucket: str = (
         ""  # for uploading reports that couldn't be sent to fastpath
     )
 
+    # Used to store history of all manifests and retrieve next manifest to use
+    anonc_manifest_bucket: str = Field(
+        default="CHANGEME",
+        description="Name of public bucket where anonymous credentials manifests are stored"
+        )
+    anonc_manifest_file: str = Field(
+        default="CHANGEME",
+        description="Name of the manifest file within the public bucket to use for ZKP verification"
+        )
+    anonc_secret_key: str = Field(
+        default="CHANGEME",
+        description="Secret key matching the specified manifest file"
+        )
+
     # ooniprobe client configuration
     collectors: List[Dict[str, str]] = [
         {"address": "httpo://guegdifjy7bjpequ.onion", "type": "onion"},

ooniapi/services/oonimeasurements/tests/migrations/0_clickhouse_init_tables.sql

@@ -65,6 +65,11 @@ ENGINE = MergeTree
 ORDER BY (report_id, input)
 SETTINGS index_granularity = 8192;
 
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `is_verified` Int8;
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `nym` Nullable(String);
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `zkp_request` Nullable(String);
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `age_range` Nullable(String);
+ALTER TABLE default.fastpath ADD COLUMN IF NOT EXISTS `msm_range` Nullable(String);
 CREATE TABLE IF NOT EXISTS default.event_detector_changepoints
 (
     `probe_asn` UInt32,
@@ -106,4 +111,4 @@ PARTITION BY toYYYYMM(ts)
 ORDER BY (probe_asn, probe_cc, ts, domain)
 SETTINGS index_granularity = 8192;
 
-ALTER TABLE default.event_detector_changepoints ADD COLUMN `block_type` String;
+ALTER TABLE default.event_detector_changepoints ADD COLUMN `block_type` String;

ooniapi/services/ooniprobe/Dockerfile

@@ -1,5 +1,5 @@
 # Python builder
-FROM python:3.11-bookworm as builder
+FROM python:3.11-bookworm AS builder
 ARG BUILD_LABEL=docker
 
 WORKDIR /build
@@ -14,10 +14,10 @@ RUN find /build -type f -name '._*' -delete
 
 RUN echo "$BUILD_LABEL" > /build/src/ooniprobe/BUILD_LABEL
 
-RUN hatch build
+RUN make build
 
 ### Actual image running on the host
-FROM python:3.11-bookworm as runner
+FROM python:3.11-bookworm AS runner
 
 WORKDIR /app
 

ooniapi/services/ooniprobe/pyproject.toml

@@ -28,7 +28,8 @@ dependencies = [
   "fastapi-utils[all] ~= 0.8.0",
   "zstd ~= 1.5.7.2",
   "boto3 ~= 1.39.3",
-  "boto3-stubs[s3] ~= 1.39.3"
+  "boto3-stubs[s3] ~= 1.39.3",
+  "ooniauth-py @ https://github.com/ooni/userauth/raw/refs/heads/release-wheels/ooniauth-py/wheels/ooniauth_py-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl"
 ]
 
 readme = "README.md"
@@ -82,6 +83,13 @@ test-cov = "pytest -s --full-trace --log-level=INFO  --log-cli-level=INFO -v --s
 cov-report = ["coverage report"]
 cov = ["test-cov", "cov-report"]
 
+# Allows specifyng dependencies as a direct reference like an URL
+# or a file path.
+# Required by ooniauth-py
+# See: https://hatch.pypa.io/1.13/config/metadata/#allowing-direct-references
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.pytest.ini_options]
 addopts = ["--import-mode=importlib"]
 

ooniapi/services/ooniprobe/scripts/new_server_state.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+"""
+Use this script to generate a new server state for the anonymous credentials protocol
+
+Two Keys will be generated:
+
+- Secret key: Save it in parameter store as a secret string, do not share it
+- Public parameters: A string that will be shared with probes to run the ZKP verification
+
+"""
+
+import ooniauth_py
+
+state = ooniauth_py.ServerState()
+
+secret_key = state.get_secret_key()
+public_params = state.get_public_parameters()
+
+print(
+    f"""
+Secret Key:
+
+    {secret_key}
+
+Public Parameters:
+
+    {public_params}
+""")