Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions .github/workflows/test_windows.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: CI

permissions: {}

on:
push:
branches: [main]
pull_request:
branches: ["**"]

jobs:
tests-windows:
name: Run tests on Windows
runs-on: windows-2022
steps:
- uses: actions/checkout@v4
with:
lfs: true
- name: Set up uv
uses: astral-sh/setup-uv@v2
with:
enable-cache: true
cache-dependency-glob: "uv.lock"
- name: "Set up Python"
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Sync
run: uv sync --all-packages --frozen
- name: Run datasets tests
run: cd tilebox-datasets && uv run --all-packages pytest -Wall -Werror -v .
- name: Run grpc tests
run: cd tilebox-grpc && uv run --all-packages pytest -Wall -Werror -v .
- name: Run storage tests
run: cd tilebox-storage && uv run --all-packages pytest -Wall -Werror -v .
- name: Run workflows tests
run: cd tilebox-workflows && uv run --all-packages pytest -Wall -Werror -v .
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ repos:
hooks:
- id: sync-with-uv
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.14.3
rev: v0.14.7
hooks:
- id: ruff-check
args: [--fix, --exit-non-zero-on-fix]
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed

- `tilebox-storage`: Fixed a bug on Windows, where the `CopernicusStorageClient` and `USGSLandsatStorageClient` were
unable to download products due to an incorrect path separator in the underlying S3 paths.

## [0.45.0] - 2025-11-17

### Added
Expand Down
17 changes: 17 additions & 0 deletions tilebox-storage/tests/test_storage_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
UmbraStorageClient,
USGSLandsatStorageClient,
_HttpClient,
list_object_paths,
)
from tilebox.storage.granule import (
ASFStorageGranule,
Expand Down Expand Up @@ -149,6 +150,22 @@ async def test_cached_download(httpx_mock: HTTPXMock, tmp_path: Path, granule: A
assert len(httpx_mock.get_requests(url=granule.urls.data)) == 1


@pytest.mark.asyncio
async def test_list_object_paths() -> None:
with TemporaryDirectory() as tmp_path:
store_path = Path(tmp_path) / "store"
store_path.mkdir(exist_ok=True, parents=True)
store = LocalStore(store_path)

await store.put_async("prefix/object1", b"content1")
await store.put_async("prefix/object2", b"content2")
await store.put_async("prefix/subdir/object3", b"content3")

objects = await list_object_paths(store, "prefix")
# we always need a forward slash in our paths, even on windows
assert objects == ["object1", "object2", "subdir/object3"]


@pytest.mark.asyncio
@given(umbra_granules())
@settings(max_examples=1, deadline=timedelta(milliseconds=100))
Expand Down
9 changes: 5 additions & 4 deletions tilebox-storage/tilebox/storage/aio.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from asyncio import Queue, QueueEmpty
from collections.abc import AsyncIterator
from pathlib import Path
from pathlib import PurePosixPath as ObjectPath
from typing import Any, TypeAlias

import anyio
Expand Down Expand Up @@ -259,8 +260,8 @@ async def destroy_cache(self) -> None:

async def list_object_paths(store: ObjectStore, prefix: str) -> list[str]:
objects = await obs.list(store, prefix).collect_async()
prefix_path = Path(prefix)
return sorted(str(Path(obj["path"]).relative_to(prefix_path)) for obj in objects)
prefix_path = ObjectPath(prefix)
return sorted(str(ObjectPath(obj["path"]).relative_to(prefix_path)) for obj in objects)


async def download_objects( # noqa: PLR0913
Expand Down Expand Up @@ -299,7 +300,7 @@ async def _download_worker(
async def _download_object(
store: ObjectStore, prefix: str, obj: str, output_dir: Path, show_progress: bool = True
) -> Path:
key = str(Path(prefix) / obj)
key = str(ObjectPath(prefix) / obj)
output_path = output_dir / obj
if output_path.exists(): # already cached
return output_path
Expand Down Expand Up @@ -609,7 +610,7 @@ async def _list_objects(self, datapoint: xr.Dataset | CopernicusStorageGranule)
granule = CopernicusStorageGranule.from_data(datapoint)
# special handling for Sentinel-5P, where the location is not a folder but a single file
if granule.location.endswith(".nc"):
return [Path(granule.granule_name).name]
return [str(ObjectPath(granule.granule_name))]

return await list_object_paths(self._store, _copernicus_s3_prefix(granule))

Expand Down
4 changes: 2 additions & 2 deletions tilebox-storage/tilebox/storage/granule.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from pathlib import PurePosixPath as ObjectPath

import xarray as xr

Expand Down Expand Up @@ -103,7 +103,7 @@ def _thumbnail_relative_to_eodata_location(thumbnail_url: str, location: str) ->
url_path = thumbnail_url.split("?path=")[-1]
url_path = url_path.removeprefix("/")
location = location.removeprefix("/eodata/")
return str(Path(url_path).relative_to(location))
return str(ObjectPath(url_path).relative_to(location))


@dataclass
Expand Down
Loading
Loading