diff --git a/dandi/cli/base.py b/dandi/cli/base.py index 30fe73247..0f54a1769 100644 --- a/dandi/cli/base.py +++ b/dandi/cli/base.py @@ -2,11 +2,82 @@ import os import click +from dandischema.conf import set_instance_config +import requests +from yarl import URL + +from dandi.consts import known_instances +from dandi.utils import ServerInfo from .. import get_logger lgr = get_logger() + +def get_server_info(dandi_id: str) -> ServerInfo: + """ + Get server info from a particular DANDI instance + + Parameters + ---------- + dandi_id : str + The ID specifying the particular known DANDI instance to query for server info. + This is a key in the `dandi.consts.known_instances` dictionary. + + Returns + ------- + ServerInfo + An object representing the server information responded by the DANDI instance. + + Raises + ------ + valueError + If the provided `dandi_id` is not a valid key in the + `dandi.consts.known_instances` dictionary. + """ + if dandi_id not in known_instances: + raise ValueError(f"Unknown DANDI instance: {dandi_id}") + + info_url = str(URL(known_instances[dandi_id].api) / "info/") + resp = requests.get(info_url) + resp.raise_for_status() + return ServerInfo.model_validate(resp.json()) + + +def bind_client(server_info: ServerInfo) -> None: + """ + Bind the DANDI client to a specific DANDI server instance. I.e., to set the DANDI + server instance as the context of subsequent command executions by the DANDI client + + Parameters + ---------- + server_info : ServerInfo + An object containing the information of the DANDI server instance to bind to. + This is typically obtained by calling `get_server_info()`. + """ + set_instance_config(server_info.instance_config) + + +def init_client(dandi_id: str) -> None: + """ + Initialize the DANDI client, including binding the client to a specific DANDI server + instance + + Parameters + ---------- + dandi_id : str + The ID specifying the particular known DANDI instance to bind the client to. + This is a key in the `dandi.consts.known_instances` dictionary. + + Raises + ------ + ValueError + If the provided `dandi_id` is not a valid key in the + `dandi.consts.known_instances` dictionary. + """ + bind_client(get_server_info(dandi_id)) + + # Aux common functionality diff --git a/dandi/cli/cmd_ls.py b/dandi/cli/cmd_ls.py index 9ac9ed42f..4c01c370a 100644 --- a/dandi/cli/cmd_ls.py +++ b/dandi/cli/cmd_ls.py @@ -3,7 +3,6 @@ import os.path as op import click -from dandischema import models from .base import devel_option, lgr, map_to_click_exceptions from .formatter import JSONFormatter, JSONLinesFormatter, PYOUTFormatter, YAMLFormatter @@ -92,6 +91,8 @@ def ls( ): """List .nwb files and dandisets metadata.""" + from dandischema import models + # TODO: more logical ordering in case of fields = None common_fields = ("path", "size") if schema is not None: diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 553ec672c..cb411130b 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -17,7 +17,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional import click -from dandischema import models from pydantic import BaseModel, Field, PrivateAttr import requests import tenacity @@ -36,7 +35,6 @@ ) from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError from .keyring import keyring_lookup, keyring_save -from .misctypes import Digest, RemoteReadableAsset from .utils import ( USER_AGENT, check_dandi_version, @@ -50,8 +48,11 @@ ) if TYPE_CHECKING: + from dandischema import models from typing_extensions import Self + from .misctypes import Digest, RemoteReadableAsset + lgr = get_logger() @@ -653,6 +654,8 @@ def check_schema_version(self, schema_version: str | None = None) -> None: uses; if not set, the schema version for the installed ``dandischema`` library is used """ + from dandischema import models + if schema_version is None: schema_version = models.get_schema_version() server_info = self.get("/info/") @@ -1065,6 +1068,8 @@ def get_metadata(self) -> models.Dandiset: metadata. Consider using `get_raw_metadata()` instead in order to fetch unstructured, possibly-invalid metadata. """ + from dandischema import models + return models.Dandiset.model_validate(self.get_raw_metadata()) def get_raw_metadata(self) -> dict[str, Any]: @@ -1469,6 +1474,8 @@ def get_metadata(self) -> models.Asset: valid metadata. Consider using `get_raw_metadata()` instead in order to fetch unstructured, possibly-invalid metadata. """ + from dandischema import models + return models.Asset.model_validate(self.get_raw_metadata()) def get_raw_metadata(self) -> dict[str, Any]: @@ -1495,6 +1502,8 @@ def get_raw_digest(self, digest_type: str | models.DigestType | None = None) -> .. versionchanged:: 0.36.0 Renamed from ``get_digest()`` to ``get_raw_digest()`` """ + from dandischema import models + if digest_type is None: digest_type = self.digest_type.value elif isinstance(digest_type, models.DigestType): @@ -1517,6 +1526,8 @@ def get_digest(self) -> Digest: a dandi-etag digest for blob resources or a dandi-zarr-checksum for Zarr resources """ + from .misctypes import Digest + algorithm = self.digest_type return Digest(algorithm=algorithm, value=self.get_raw_digest(algorithm)) @@ -1651,6 +1662,8 @@ def digest_type(self) -> models.DigestType: determined based on its underlying data: dandi-etag for blob resources, dandi-zarr-checksum for Zarr resources """ + from dandischema import models + if self.asset_type is AssetType.ZARR: return models.DigestType.dandi_zarr_checksum else: @@ -1683,6 +1696,8 @@ def as_readable(self) -> RemoteReadableAsset: Returns a `Readable` instance that can be used to obtain a file-like object for reading bytes directly from the asset on the server """ + from .misctypes import RemoteReadableAsset + md = self.get_raw_metadata() local_prefix = self.client.api_url.lower() for url in md.get("contentUrl", []): @@ -1965,6 +1980,10 @@ def from_server_data( cls, asset: BaseRemoteZarrAsset, data: ZarrEntryServerData ) -> RemoteZarrEntry: """:meta private:""" + from dandischema import models + + from .misctypes import Digest + return cls( client=asset.client, zarr_id=asset.zarr, diff --git a/dandi/dandiset.py b/dandi/dandiset.py index 63ebbfb1b..bce4a818d 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -7,8 +7,6 @@ from pathlib import Path, PurePath, PurePosixPath from typing import TYPE_CHECKING -from dandischema.models import get_schema_version - from . import get_logger from .consts import dandiset_metadata_file from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files @@ -32,6 +30,8 @@ def __init__( schema_version: str | None = None, ) -> None: if schema_version is not None: + from dandischema.models import get_schema_version + current_version = get_schema_version() if schema_version != current_version: raise ValueError( diff --git a/dandi/download.py b/dandi/download.py index 51cf31de3..874b33b28 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -21,7 +21,6 @@ from typing import IO, Any, Literal from dandischema.digests.dandietag import ETagHashlike -from dandischema.models import DigestType from fasteners import InterProcessLock import humanize from interleave import FINISH_CURRENT, lazy_interleave @@ -995,6 +994,8 @@ def digest_callback(path: str, algoname: str, d: str) -> None: digests[path] = d def downloads_gen(): + from dandischema.models import DigestType + for entry in asset.iterfiles(): entries.append(entry) etag = entry.digest diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 15bc616c4..d1c547959 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -10,15 +10,12 @@ from pathlib import Path import re from threading import Lock -from typing import IO, Any, Generic +from typing import IO, TYPE_CHECKING, Any, Generic, TypeVar from xml.etree.ElementTree import fromstring import dandischema from dandischema.consts import DANDI_SCHEMA_VERSION from dandischema.digests.dandietag import DandiETag -from dandischema.models import BareAsset, CommonModel -from dandischema.models import Dandiset as DandisetMeta -from dandischema.models import get_schema_version from packaging.version import Version from pydantic import ValidationError from pydantic_core import ErrorDetails @@ -27,7 +24,6 @@ import dandi from dandi.dandiapi import RemoteAsset, RemoteDandiset, RESTFullAPIClient from dandi.metadata.core import get_default_metadata -from dandi.misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, P from dandi.utils import post_upload_size_check, pre_upload_size_check, yaml_load from dandi.validate_types import ( ORIGIN_INTERNAL_DANDI, @@ -41,6 +37,15 @@ Validator, ) +if TYPE_CHECKING: + from dandischema.models import BareAsset, CommonModel + from dandischema.models import Dandiset as DandisetMeta + + # noinspection PyUnresolvedReferences + from dandi.misctypes import BasePath, Digest, LocalReadableFile + +P = TypeVar("P", bound="BasePath") + lgr = dandi.get_logger() # TODO -- should come from schema. This is just a simplistic example for now @@ -107,6 +112,8 @@ def get_metadata( ignore_errors: bool = True, ) -> DandisetMeta: """Return the Dandiset metadata inside the file""" + from dandischema.models import Dandiset as DandisetMeta + with open(self.filepath) as f: meta = yaml_load(f, typ="safe") return DandisetMeta.model_construct(**meta) @@ -126,6 +133,9 @@ def get_validation_errors( meta, _required_dandiset_metadata_fields, str(self.filepath) ) else: + from dandischema.models import Dandiset as DandisetMeta + from dandischema.models import get_schema_version + current_version = get_schema_version() if schema_version != current_version: raise ValueError( @@ -147,6 +157,8 @@ def as_readable(self) -> LocalReadableFile: Returns a `Readable` instance wrapping the local file """ + from dandi.misctypes import LocalReadableFile + return LocalReadableFile(self.filepath) @@ -161,7 +173,11 @@ class LocalAsset(DandiFile): #: (i.e., relative to the Dandiset's root) path: str - _DUMMY_DIGEST = DUMMY_DANDI_ETAG + @staticmethod + def _get_dummy_digest() -> Digest: + from dandi.misctypes import get_dummy_dandi_etag + + return get_dummy_dandi_etag() @abstractmethod def get_digest(self) -> Digest: @@ -186,6 +202,8 @@ def get_validation_errors( schema_version: str | None = None, devel_debug: bool = False, ) -> list[ValidationResult]: + from dandischema.models import BareAsset, get_schema_version + current_version = get_schema_version() if schema_version is None: schema_version = current_version @@ -194,7 +212,7 @@ def get_validation_errors( f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: - asset = self.get_metadata(digest=self._DUMMY_DIGEST) + asset = self.get_metadata(digest=self._get_dummy_digest()) BareAsset(**asset.model_dump()) except ValidationError as e: if devel_debug: @@ -309,6 +327,7 @@ def get_metadata( def get_digest(self) -> Digest: """Calculate a dandi-etag digest for the asset""" # Avoid heavy import by importing within function: + from dandi.misctypes import Digest from dandi.support.digests import get_digest value = get_digest(self.filepath, digest="dandi-etag") @@ -476,6 +495,8 @@ def as_readable(self) -> LocalReadableFile: Returns a `Readable` instance wrapping the local file """ + from dandi.misctypes import LocalReadableFile + return LocalReadableFile(self.filepath) diff --git a/dandi/files/bids.py b/dandi/files/bids.py index a441e5999..39c8d713a 100644 --- a/dandi/files/bids.py +++ b/dandi/files/bids.py @@ -5,10 +5,9 @@ from datetime import datetime from pathlib import Path from threading import Lock +from typing import TYPE_CHECKING import weakref -from dandischema.models import BareAsset - from dandi.bids_validator_deno import bids_validate from .bases import GenericAsset, LocalFileAsset, NWBAsset @@ -18,6 +17,9 @@ from ..misctypes import Digest from ..validate_types import ValidationResult +if TYPE_CHECKING: + from dandischema.models import BareAsset + BIDS_ASSET_ERRORS = ("BIDS.NON_BIDS_PATH_PLACEHOLDER",) BIDS_DATASET_ERRORS = ("BIDS.MANDATORY_FILE_MISSING_PLACEHOLDER",) @@ -84,6 +86,8 @@ def _get_metadata(self) -> None: This populates `self._asset_metadata` """ + from dandischema.models import BareAsset + with self._lock: if self._asset_metadata is None: # Import here to avoid circular import @@ -236,6 +240,8 @@ def get_metadata( digest: Digest | None = None, ignore_errors: bool = True, ) -> BareAsset: + from dandischema.models import BareAsset + bids_metadata = BIDSAsset.get_metadata(self, digest, ignore_errors) nwb_metadata = NWBAsset.get_metadata(self, digest, ignore_errors) return BareAsset( diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 6220e2294..42077d06e 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -11,9 +11,8 @@ import os.path from pathlib import Path from time import sleep -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional -from dandischema.models import BareAsset, DigestType from pydantic import BaseModel, ConfigDict, ValidationError import requests from zarr_checksum.tree import ZarrChecksumTree @@ -34,7 +33,7 @@ RESTFullAPIClient, ) from dandi.metadata.core import get_default_metadata -from dandi.misctypes import DUMMY_DANDI_ZARR_CHECKSUM, BasePath, Digest +from dandi.misctypes import BasePath, Digest from dandi.utils import ( chunked, exclude_from_zarr, @@ -55,6 +54,9 @@ Validator, ) +if TYPE_CHECKING: + from dandischema.models import BareAsset + lgr = get_logger() @@ -320,6 +322,8 @@ def get_digest(self) -> Digest: directory, the algorithm will be the DANDI Zarr checksum algorithm; if it is a file, it will be MD5. """ + from dandischema.models import DigestType + # Avoid heavy import by importing within function: from dandi.support.digests import get_digest, get_zarr_checksum @@ -363,7 +367,11 @@ class ZarrStat: class ZarrAsset(LocalDirectoryAsset[LocalZarrEntry]): """Representation of a local Zarr directory""" - _DUMMY_DIGEST = DUMMY_DANDI_ZARR_CHECKSUM + @staticmethod + def _get_dummy_digest() -> Digest: + from dandi.misctypes import get_dummy_dandi_zarr_checksum + + return get_dummy_dandi_zarr_checksum() @property def filetree(self) -> LocalZarrEntry: diff --git a/dandi/metadata/core.py b/dandi/metadata/core.py index 12a53264e..b77afabe0 100644 --- a/dandi/metadata/core.py +++ b/dandi/metadata/core.py @@ -3,21 +3,26 @@ from datetime import datetime from pathlib import Path import re +from typing import TYPE_CHECKING -from dandischema import models from pydantic import ByteSize -from .util import extract_model, get_generator from .. import get_logger -from ..misctypes import Digest, LocalReadableFile, Readable from ..utils import get_mime_type, get_utcnow_datetime +if TYPE_CHECKING: + from dandischema import models + + from ..misctypes import Digest, Readable + lgr = get_logger() def get_default_metadata( path: str | Path | Readable, digest: Digest | None = None ) -> models.BareAsset: + from dandischema import models + metadata = models.BareAsset.model_construct() # type: ignore[call-arg] start_time = end_time = datetime.now().astimezone() add_common_metadata(metadata, path, start_time, end_time, digest) @@ -35,6 +40,11 @@ def add_common_metadata( Update a `dict` of raw "schemadata" with the fields that are common to both NWB assets and non-NWB assets """ + from dandischema import models + + from .util import get_generator + from ..misctypes import LocalReadableFile, Readable + if digest is not None: metadata.digest = digest.asdict() else: @@ -73,4 +83,8 @@ def prepare_metadata(metadata: dict) -> models.BareAsset: .. [2] metadata in the form used by the ``dandischema`` library """ + from dandischema import models + + from .util import extract_model + return extract_model(models.BareAsset, metadata) diff --git a/dandi/metadata/nwb.py b/dandi/metadata/nwb.py index 38c2ac4e7..1413e2917 100644 --- a/dandi/metadata/nwb.py +++ b/dandi/metadata/nwb.py @@ -13,7 +13,7 @@ from .. import get_logger from ..consts import metadata_all_fields from ..files import bids, dandi_file, find_bids_dataset_description -from ..misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, Readable +from ..misctypes import Digest, LocalReadableFile, Readable, get_dummy_dandi_etag from ..pynwb_utils import ( _get_pynwb_metadata, get_neurodata_types, @@ -68,7 +68,7 @@ def get_metadata( ) assert isinstance(df, bids.BIDSAsset) if not digest: - digest = DUMMY_DANDI_ETAG + digest = get_dummy_dandi_etag() path_metadata = df.get_metadata(digest=digest) meta["bids_version"] = df.get_validation_bids_version() # there might be a more elegant way to do this: diff --git a/dandi/misctypes.py b/dandi/misctypes.py index 9ed2dcf07..8df0aa283 100644 --- a/dandi/misctypes.py +++ b/dandi/misctypes.py @@ -11,11 +11,13 @@ from dataclasses import dataclass from datetime import datetime from fnmatch import fnmatchcase +from functools import cache import os.path from pathlib import Path -from typing import IO, TypeVar, cast +from typing import IO, TYPE_CHECKING, TypeVar, cast -from dandischema.models import DigestType +if TYPE_CHECKING: + from dandischema.models import DigestType @dataclass @@ -34,6 +36,8 @@ def dandi_etag(cls, value: str) -> Digest: Construct a `Digest` with the given value and a ``algorithm`` of ``DigestType.dandi_etag`` """ + from dandischema.models import DigestType + return cls(algorithm=DigestType.dandi_etag, value=value) @classmethod @@ -42,6 +46,8 @@ def dandi_zarr(cls, value: str) -> Digest: Construct a `Digest` with the given value and a ``algorithm`` of ``DigestType.dandi_zarr_checksum`` """ + from dandischema.models import DigestType + return cls(algorithm=DigestType.dandi_zarr_checksum, value=value) def asdict(self) -> dict[DigestType, str]: @@ -54,11 +60,19 @@ def asdict(self) -> dict[DigestType, str]: #: Placeholder digest used in some situations where a digest is required but #: not actually relevant and would be too expensive to calculate -DUMMY_DANDI_ETAG = Digest(algorithm=DigestType.dandi_etag, value=32 * "d" + "-1") -DUMMY_DANDI_ZARR_CHECKSUM = Digest( - algorithm=DigestType.dandi_zarr_checksum, - value=32 * "d" + "-1--1", -) +@cache +def get_dummy_dandi_etag() -> Digest: + from dandischema.models import DigestType + + return Digest(algorithm=DigestType.dandi_etag, value=32 * "d" + "-1") + + +@cache +def get_dummy_dandi_zarr_checksum() -> Digest: + from dandischema.models import DigestType + + return Digest(algorithm=DigestType.dandi_zarr_checksum, value=32 * "d" + "-1--1") + P = TypeVar("P", bound="BasePath") diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 5c20acf0d..2bd2ff0fb 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -49,7 +49,7 @@ species_map, timedelta2duration, ) -from ..misctypes import DUMMY_DANDI_ETAG +from ..misctypes import get_dummy_dandi_etag from ..utils import ensure_datetime METADATA_DIR = Path(__file__).with_name("data") / "metadata" @@ -836,7 +836,9 @@ def test_ndtypes(ndtypes, asset_dict): def test_nwb2asset(simple2_nwb: Path) -> None: # Classes with ANY_AWARE_DATETIME fields need to be constructed with # model_construct() - assert nwb2asset(simple2_nwb, digest=DUMMY_DANDI_ETAG) == BareAsset.model_construct( + assert nwb2asset( + simple2_nwb, digest=get_dummy_dandi_etag() + ) == BareAsset.model_construct( schemaKey="Asset", schemaVersion=DANDI_SCHEMA_VERSION, keywords=["keyword1", "keyword 2"], diff --git a/dandi/utils.py b/dandi/utils.py index 06e461fbc..caeac7ac2 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -26,6 +26,7 @@ import types from typing import IO, Any, List, Optional, Protocol, TypeVar, Union +from dandischema.conf import Config as InstanceConfig import dateutil.parser from multidict import MultiDict # dependency of yarl from pydantic import BaseModel, Field @@ -546,6 +547,7 @@ class ServerServices(BaseModel): class ServerInfo(BaseModel): + instance_config: InstanceConfig # schema_version: str # schema_url: str version: str diff --git a/setup.cfg b/setup.cfg index 5efe08400..9c3240a80 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = # >=8.2.0: https://github.com/pallets/click/issues/2911 click >= 7.1, <8.2.0 click-didyoumean - dandischema >= 0.11.0, < 0.12.0 + dandischema @ git+https://github.com/dandi/dandi-schema.git@devendorize etelemetry >= 0.2.2 # For pydantic to be able to use type annotations like `X | None` eval_type_backport; python_version < "3.10"