From 2926ba0287e0c34e7889d8723d56ef3d4507e197 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 30 Oct 2025 13:41:26 -0400 Subject: [PATCH 1/5] Allow for client to provide records with newer version to the server Server might need then to either wait until upgrade it becomes capable of validating them or we might in the future to allow client to downgrade them (on client; loselessly only) since server cannot yet handle them. This behavior would facilitate testing of new releases of dandi-schema which now would fail since we require 1-to-1 correspondence. See e.g. https://github.com/dandi/dandi-schema/pull/342#issuecomment-3453091185 --- dandi/dandiapi.py | 33 +++++++++++-- dandi/tests/test_dandiapi.py | 89 ++++++++++++++++++++++-------------- 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 5834b7513..378b6e01d 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -18,6 +18,7 @@ import click from dandischema import models +from packaging.version import Version as PackagingVersion from pydantic import BaseModel, Field, PrivateAttr import requests import tenacity @@ -646,8 +647,12 @@ def create_dandiset( def check_schema_version(self, schema_version: str | None = None) -> None: """ - Confirms that the server is using the same version of the DANDI schema - as the client. If it is not, a `SchemaVersionError` is raised. + Confirms that the server is using a "compatible" version of the DANDI schema. + Compatibility here means that the server's schema version can be lower than + client has, but within the same MAJOR.MINOR component of the version number + for 0.x series, and same MAJOR version for/after 1.x series. + + If it is not, a `SchemaVersionError` is raised. :param schema_version: the schema version to confirm that the server uses; if not set, the schema version for the installed @@ -662,12 +667,30 @@ def check_schema_version(self, schema_version: str | None = None) -> None: "Server did not provide schema_version in /info/;" f" returned {server_info!r}" ) - if server_schema_version != schema_version: + server_ver, our_ver = PackagingVersion(server_schema_version), PackagingVersion( + schema_version + ) + if server_ver > our_ver: raise SchemaVersionError( - f"Server requires schema version {server_schema_version};" - f" client only supports {schema_version}. You may need to" + f"Server uses schema version {server_schema_version};" + f" client only supports prior {schema_version}. You may need to" " upgrade dandi and/or dandischema." ) + # NOTE: here we could in theory support older schema versions as long as they + # could be migrated, but client would not know how to migrate -- server "might" + # as it is the one having newer version! + # TODO: check current server behavior which is likely to just not care! + # So that is where server might need to provide support for upgrades upon + # providing metadata. + elif ( + server_ver.major == 0 and server_ver.release[:2] != our_ver.release[:2] + ) or ( + server_ver.major != our_ver.major + ): # MAJOR, MINOR within 0.x.y and MAJOR within 1.x.y + raise SchemaVersionError( + f"Server uses older incompatible schema version {server_schema_version};" + f" client supports {schema_version}." + ) def get_asset(self, asset_id: str) -> BaseRemoteAsset: """ diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index d445ea90f..e42fa11b8 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -272,35 +272,56 @@ def test_remote_asset_json_dict(text_dandiset: SampleDandiset) -> None: } +@pytest.mark.parametrize( + "server_schema_version,local_schema_version,should_raise,expected_message_start", + [ + # Current/identity matching -- always ok + (get_schema_version(), None, False, None), + (get_schema_version(), get_schema_version(), False, None), + ("0.6.7", "0.6.7", False, None), + # Less - is not good since we might be missing fields and it is easy + # for client to upgrade. + ( + "4.5.6", + "4.5.5", + True, + "Server uses schema version 4.5.6; client only supports prior 4.5.5. " + "You may need to upgrade dandi and/or dandischema.", + ), + ( + "0.6.7", + "0.6.6", + True, + "Server uses schema version 0.6.7; client only supports prior 0.6.6", + ), + # Now - incompatible, for 0.x -- rely on MAJOR.MINOR + ( + "0.6.7", + "0.7.0", + True, + "Server uses older incompatible schema version 0.6.7; client supports 0.7.0", + ), + # After 1.x -- rely on MAJOR. + ("1.0.0", "1.2.3", False, None), + ("1.6.7", "1.7.0", False, None), + ("1.6.7", "1.8.3", False, None), + ( + "1.6.7", + "2.7.0", + True, + "Server uses older incompatible schema version 1.6.7; client supports 2.7.0", + ), + ], +) @responses.activate -def test_check_schema_version_matches_default() -> None: - server_info = { - "schema_version": get_schema_version(), - "version": "0.0.0", - "services": { - "api": {"url": "https://test.nil/api"}, - }, - "cli-minimal-version": "0.0.0", - "cli-bad-versions": [], - } - responses.add( - responses.GET, - "https://test.nil/server-info", - json=server_info, - ) - responses.add( - responses.GET, - "https://test.nil/api/info/", - json=server_info, - ) - client = DandiAPIClient("https://test.nil/api") - client.check_schema_version() - - -@responses.activate -def test_check_schema_version_mismatch() -> None: +def test_check_schema_version( + server_schema_version: str, + local_schema_version: str | None, + should_raise: bool, + expected_message_start: str | None, +) -> None: server_info = { - "schema_version": "4.5.6", + "schema_version": server_schema_version, "version": "0.0.0", "services": { "api": {"url": "https://test.nil/api"}, @@ -319,13 +340,13 @@ def test_check_schema_version_mismatch() -> None: json=server_info, ) client = DandiAPIClient("https://test.nil/api") - with pytest.raises(SchemaVersionError) as excinfo: - client.check_schema_version("1.2.3") - assert ( - str(excinfo.value) - == "Server requires schema version 4.5.6; client only supports 1.2.3. " - "You may need to upgrade dandi and/or dandischema." - ) + if should_raise: + with pytest.raises(SchemaVersionError) as excinfo: + client.check_schema_version(local_schema_version) + if expected_message_start: + assert str(excinfo.value).startswith(expected_message_start) + else: + client.check_schema_version(local_schema_version) def test_get_dandisets(text_dandiset: SampleDandiset) -> None: From 4e1c7dbbcd6375410c07a458064d462738755ace Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 30 Oct 2025 16:17:00 -0400 Subject: [PATCH 2/5] Allow older clients to upload as long as it is a version known to be upgradable ATM server DOES NOT CARE! So we already allow for assets being created by other clients with outdated schema etc. They fail validation unless could be upgraded! Changes might need to be adjusted based on the name for the /info record field as would be decided in https://github.com/dandi/dandi-archive/issues/2624 or PR to solve it --- dandi/dandiapi.py | 34 +++++++++++++++++++++++----------- dandi/tests/test_dandiapi.py | 14 ++++++++++---- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 378b6e01d..dcdcd3f7c 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -18,6 +18,7 @@ import click from dandischema import models +import dandischema.consts from packaging.version import Version as PackagingVersion from pydantic import BaseModel, Field, PrivateAttr import requests @@ -648,11 +649,16 @@ def create_dandiset( def check_schema_version(self, schema_version: str | None = None) -> None: """ Confirms that the server is using a "compatible" version of the DANDI schema. - Compatibility here means that the server's schema version can be lower than - client has, but within the same MAJOR.MINOR component of the version number - for 0.x series, and same MAJOR version for/after 1.x series. - If it is not, a `SchemaVersionError` is raised. + Compatibility here means that the server's schema version can be either + + - lower than client has, but within the same MAJOR.MINOR component of the version + number for 0.x series, and same MAJOR version for/after 1.x series. + - higher than client has, but only if the client's schema version is listed + among the server's `allowed_schema_versions` (as returned by the `/info` API endpoint), + or if not there -- `dandischema.consts.ALLOWED_INPUT_SCHEMAS` is consulted. + + If neither of above, a `SchemaVersionError` is raised. :param schema_version: the schema version to confirm that the server uses; if not set, the schema version for the installed @@ -671,14 +677,20 @@ def check_schema_version(self, schema_version: str | None = None) -> None: schema_version ) if server_ver > our_ver: - raise SchemaVersionError( - f"Server uses schema version {server_schema_version};" - f" client only supports prior {schema_version}. You may need to" - " upgrade dandi and/or dandischema." + # TODO: potentially adjust here if name would be different: see + # https://github.com/dandi/dandi-archive/issues/2624 + allowed_schema_versions = server_info.get( + "allowed_schema_versions", dandischema.consts.ALLOWED_INPUT_SCHEMAS ) - # NOTE: here we could in theory support older schema versions as long as they - # could be migrated, but client would not know how to migrate -- server "might" - # as it is the one having newer version! + if schema_version not in allowed_schema_versions: + raise SchemaVersionError( + f"Server uses schema version {server_schema_version};" + f" client only supports prior {schema_version} and it" + f" is not among any of the allowed upgradable schema versions" + f" ({', '.join(allowed_schema_versions)}) . You may need to" + " upgrade dandi and/or dandischema." + ) + # TODO: check current server behavior which is likely to just not care! # So that is where server might need to provide support for upgrades upon # providing metadata. diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index e42fa11b8..848c6f6eb 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -285,14 +285,20 @@ def test_remote_asset_json_dict(text_dandiset: SampleDandiset) -> None: "4.5.6", "4.5.5", True, - "Server uses schema version 4.5.6; client only supports prior 4.5.5. " - "You may need to upgrade dandi and/or dandischema.", + "Server uses schema version 4.5.6; client only supports prior 4.5.5 " + "and it is not among any of the allowed upgradable schema versions", ), ( "0.6.7", - "0.6.6", + "0.3.0", True, - "Server uses schema version 0.6.7; client only supports prior 0.6.6", + "Server uses schema version 0.6.7; client only supports prior 0.3.0", + ), + ( + "0.6.7", + "0.6.6", # can be upgraded and thus uploaded! + False, + None, ), # Now - incompatible, for 0.x -- rely on MAJOR.MINOR ( From 9e2f85242c8f74066e14b2487fcf7a0e577d8010 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 30 Oct 2025 19:59:36 -0400 Subject: [PATCH 3/5] Tune ups to docstrings from code review Co-authored-by: Isaac To --- dandi/dandiapi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index dcdcd3f7c..ecabfc628 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -648,20 +648,20 @@ def create_dandiset( def check_schema_version(self, schema_version: str | None = None) -> None: """ - Confirms that the server is using a "compatible" version of the DANDI schema. + Confirms that the the given schema version at the client is "compatible" the server. Compatibility here means that the server's schema version can be either - lower than client has, but within the same MAJOR.MINOR component of the version number for 0.x series, and same MAJOR version for/after 1.x series. - - higher than client has, but only if the client's schema version is listed + - otherwise, but only if the client's schema version is listed among the server's `allowed_schema_versions` (as returned by the `/info` API endpoint), or if not there -- `dandischema.consts.ALLOWED_INPUT_SCHEMAS` is consulted. If neither of above, a `SchemaVersionError` is raised. - :param schema_version: the schema version to confirm that the server - uses; if not set, the schema version for the installed + :param schema_version: the schema version to be confirmed for compatibility with the server; + if not set, the schema version for the installed ``dandischema`` library is used """ if schema_version is None: From 9034dfacf3b5a88b06113d6e3c0b901059c82327 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 30 Oct 2025 20:06:51 -0400 Subject: [PATCH 4/5] Fixup wording and formatting a little more --- dandi/dandiapi.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index ecabfc628..1a639eb26 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -648,21 +648,21 @@ def create_dandiset( def check_schema_version(self, schema_version: str | None = None) -> None: """ - Confirms that the the given schema version at the client is "compatible" the server. + Confirms that the given schema version at the client is "compatible" with the server. Compatibility here means that the server's schema version can be either - lower than client has, but within the same MAJOR.MINOR component of the version - number for 0.x series, and same MAJOR version for/after 1.x series. - - otherwise, but only if the client's schema version is listed + number for 0.x series, and same MAJOR version for/after 1.x series; + - the same; + - higher than the client has, but only if the client's schema version is listed among the server's `allowed_schema_versions` (as returned by the `/info` API endpoint), or if not there -- `dandischema.consts.ALLOWED_INPUT_SCHEMAS` is consulted. If neither of above, a `SchemaVersionError` is raised. - :param schema_version: the schema version to be confirmed for compatibility with the server; - if not set, the schema version for the installed - ``dandischema`` library is used + :param schema_version: the schema version to be confirmed for compatibility with the server; + if not set, the schema version for the installed ``dandischema`` library is used. """ if schema_version is None: schema_version = models.get_schema_version() From 0a0c59c9b21e6095813353878b3f4858502e5de6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 31 Oct 2025 11:44:46 -0400 Subject: [PATCH 5/5] Warn about using newer schema version while uploading to the server Running tests with --log-cli-level=WARNING shows: dandi/tests/test_dandiapi.py::test_check_schema_version[1.0.0-1.2.3-False-None] ------------------------------------------------------------------ live log call ------------------------------------------------------------------- WARNING dandi:dandiapi.py:710 Server uses schema version 1.0.0 older than client's 1.2.3 (dandischema library 0.11.1). Server might fail to validate such assets and you might not be able to publish this dandiset until server is upgraded. Alternatively, you may downgrade dandischema and reupload. PASSED dandi/tests/test_dandiapi.py::test_check_schema_version[1.6.7-1.7.0-False-None] ------------------------------------------------------------------ live log call ------------------------------------------------------------------- WARNING dandi:dandiapi.py:710 Server uses schema version 1.6.7 older than client's 1.7.0 (dandischema library 0.11.1). Server might fail to validate such assets and you might not be able to publish this dandiset until server is upgraded. Alternatively, you may downgrade dandischema and reupload. PASSED dandi/tests/test_dandiapi.py::test_check_schema_version[1.6.7-1.8.3-False-None] ------------------------------------------------------------------ live log call ------------------------------------------------------------------- WARNING dandi:dandiapi.py:710 Server uses schema version 1.6.7 older than client's 1.8.3 (dandischema library 0.11.1). Server might fail to validate such assets and you might not be able to publish this dandiset until server is upgraded. Alternatively, you may downgrade dandischema and reupload. PASSED dandi/tests/test_dandiapi.py::test_check_schema_version[1.6.7-2.7.0-True-Server uses older incompatible schema version 1.6.7; client supports 2.7.0] PASSED I think it is informative enough! --- dandi/dandiapi.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 1a639eb26..37343c444 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -703,6 +703,19 @@ def check_schema_version(self, schema_version: str | None = None) -> None: f"Server uses older incompatible schema version {server_schema_version};" f" client supports {schema_version}." ) + elif server_ver < our_ver: + # Compatible older server version -- all good, but inform the user + # TODO: potentially downgrade the record to match the schema, + # see https://github.com/dandi/dandi-schema/issues/343 + lgr.warning( + "Server uses schema version %s older than client's %s (dandischema library %s). " + "Server might fail to validate such assets and you might not be able to " + "publish this dandiset until server is upgraded. " + "Alternatively, you may downgrade dandischema and reupload.", + server_ver, + our_ver, + dandischema.__version__, + ) def get_asset(self, asset_id: str) -> BaseRemoteAsset: """