From b13c043f57bde5b1540b326489f6748d1fe0c2dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:15:07 +0000 Subject: [PATCH 1/4] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.16.4 → v0.16.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.4...v0.16.5) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 944cf73c..06813bdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: language: system pass_filenames: false - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.4 + rev: v0.16.5 hooks: - id: ruff args: ["--fix"] From 14d23a5a43d6a414f900de607b43ed316fda65a1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:15:17 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 93f6dd03..6a2e8b20 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ To use the project, simply install our package `zarrs` from PyPI (which depends ```python import zarr + zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"}) ``` @@ -58,19 +59,21 @@ The `ZarrsCodecPipeline` specific options are: For example: ```python -zarr.config.set({ - "threading.max_workers": None, - "array.write_empty_chunks": False, - "codec_pipeline": { - "path": "zarrs.ZarrsCodecPipeline", - "validate_checksums": True, - "chunk_concurrent_maximum": None, - "chunk_concurrent_minimum": 4, - "file_handle_cache_size": 0, - "direct_io": False, - "strict": False +zarr.config.set( + { + "threading.max_workers": None, + "array.write_empty_chunks": False, + "codec_pipeline": { + "path": "zarrs.ZarrsCodecPipeline", + "validate_checksums": True, + "chunk_concurrent_maximum": None, + "chunk_concurrent_minimum": 4, + "file_handle_cache_size": 0, + "direct_io": False, + "strict": False, + }, } -}) +) ``` If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value. However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed. This may change in the future as guidance from the `zarr` community becomes clear. From c0d9ac67ae363d916d541be7ee225acb78421792 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Mon, 31 Aug 2026 21:37:27 +0200 Subject: [PATCH 3/4] preview --- .pre-commit-config.yaml | 1 + README.md | 28 +++++++++++++--------------- src/chunk_item.rs | 2 +- src/lib.rs | 30 ++++++++++++++---------------- src/store.rs | 2 +- src/store/filesystem.rs | 2 +- tests/conftest.py | 3 ++- tests/pipeline/conftest.py | 14 ++++++-------- tests/pipeline/test_pipeline.py | 30 ++++++++++++------------------ tests/pipeline/test_roundtrip.py | 6 +++--- tests/test_zarrs_http.py | 22 ++++++++++------------ 11 files changed, 64 insertions(+), 76 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06813bdb..a1c8f2bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,7 @@ repos: - id: ruff args: ["--fix"] - id: ruff-format + args: ["--preview"] # The following can be removed once PLR0917 is out of preview - name: ruff preview rules id: ruff diff --git a/README.md b/README.md index 6a2e8b20..9300ccc1 100644 --- a/README.md +++ b/README.md @@ -59,21 +59,19 @@ The `ZarrsCodecPipeline` specific options are: For example: ```python -zarr.config.set( - { - "threading.max_workers": None, - "array.write_empty_chunks": False, - "codec_pipeline": { - "path": "zarrs.ZarrsCodecPipeline", - "validate_checksums": True, - "chunk_concurrent_maximum": None, - "chunk_concurrent_minimum": 4, - "file_handle_cache_size": 0, - "direct_io": False, - "strict": False, - }, - } -) +zarr.config.set({ + "threading.max_workers": None, + "array.write_empty_chunks": False, + "codec_pipeline": { + "path": "zarrs.ZarrsCodecPipeline", + "validate_checksums": True, + "chunk_concurrent_maximum": None, + "chunk_concurrent_minimum": 4, + "file_handle_cache_size": 0, + "direct_io": False, + "strict": False, + }, +}) ``` If the `ZarrsCodecPipeline` is pickled, and then un-pickled, and during that time one of `chunk_concurrent_minimum`, `chunk_concurrent_maximum`, or `num_threads` has changed, the newly un-pickled version will pick up the new value. However, once a `ZarrsCodecPipeline` object has been instantiated, these values are then fixed. This may change in the future as guidance from the `zarr` community becomes clear. diff --git a/src/chunk_item.rs b/src/chunk_item.rs index 8a9aa0ad..c7993e95 100644 --- a/src/chunk_item.rs +++ b/src/chunk_item.rs @@ -26,7 +26,7 @@ fn to_nonzero_u64_vec(v: Vec) -> PyResult> { #[derive(Clone)] #[gen_stub_pyclass] #[pyclass] -pub(crate) struct ChunkItem { +pub struct ChunkItem { pub key: StoreKey, pub chunk_subset: ArraySubset, pub subset: ArraySubset, diff --git a/src/lib.rs b/src/lib.rs index 3ce7eb2e..8424b074 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -257,17 +257,15 @@ impl CodecPipelineImpl { DataType::from_metadata(&metadata_v3.data_type).map_py_err::()?; let fill_value = data_type .fill_value(&metadata_v3.fill_value, ZarrVersion::V3) - .or_else(|_| { - Err(match &metadata { - ArrayMetadata::V2(metadata) => format!( - "incompatible fill value metadata: dtype={}, fill_value={}", - metadata.dtype, metadata.fill_value - ), - ArrayMetadata::V3(metadata) => format!( - "incompatible fill value metadata: data_type={}, fill_value={}", - metadata.data_type, metadata.fill_value - ), - }) + .map_err(|_| match &metadata { + ArrayMetadata::V2(metadata) => format!( + "incompatible fill value metadata: dtype={}, fill_value={}", + metadata.dtype, metadata.fill_value + ), + ArrayMetadata::V3(metadata) => format!( + "incompatible fill value metadata: data_type={}, fill_value={}", + metadata.data_type, metadata.fill_value + ), }) .map_py_err::()?; @@ -469,11 +467,11 @@ impl CodecPipelineImpl { /// A Python module implemented in Rust. #[pymodule] -fn _internal(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add("__version__", env!("CARGO_PKG_VERSION"))?; - m.add_class::()?; - m.add_class::()?; - Ok(()) +pub mod _internal { + #[allow(non_upper_case_globals)] + pub const __version__: &str = env!("CARGO_PKG_VERSION"); + pub use super::CodecPipelineImpl; + pub use super::chunk_item::ChunkItem; } define_stub_info_gatherer!(stub_info); diff --git a/src/store.rs b/src/store.rs index 335c77bd..0176fa85 100644 --- a/src/store.rs +++ b/src/store.rs @@ -72,7 +72,7 @@ impl<'py> FromPyObject<'_, 'py> for StoreConfig { } impl StoreConfig { - pub fn direct_io(&mut self, flag: bool) -> () { + pub fn direct_io(&mut self, flag: bool) { match self { StoreConfig::Filesystem(config) => config.direct_io(flag), StoreConfig::Http(_config) => (), diff --git a/src/store/filesystem.rs b/src/store/filesystem.rs index 0eb59212..37dd8c1c 100644 --- a/src/store/filesystem.rs +++ b/src/store/filesystem.rs @@ -22,7 +22,7 @@ impl FilesystemStoreConfig { } } - pub fn direct_io(&mut self, flag: bool) -> () { + pub fn direct_io(&mut self, flag: bool) { self.opts.direct_io(flag); } diff --git a/tests/conftest.py b/tests/conftest.py index 33cb1295..6451a656 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,8 @@ async def store(request: pytest.FixtureRequest, tmp_path: Path) -> Store: def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]: array_request: ArrayRequest = request.param return ( - np.arange(np.prod(array_request.shape)) + np + .arange(np.prod(array_request.shape)) .reshape(array_request.shape, order=array_request.order) .astype(array_request.dtype) ) diff --git a/tests/pipeline/conftest.py b/tests/pipeline/conftest.py index 8afbe77f..4259ac8e 100644 --- a/tests/pipeline/conftest.py +++ b/tests/pipeline/conftest.py @@ -113,14 +113,12 @@ def roundtrip_params() -> Generator[ParameterSet]: if sum(isinstance(i, EllipsisType) for i in index) > 1: continue for indexing_method_param in indexing_method_params: - id = "-".join( - [ - str(indexing_method_param.id), - f"{dimensionality}d", - *(str(index_param.id) for index_param in index_param_prod), - f"v{format}", - ] - ) + id = "-".join([ + str(indexing_method_param.id), + f"{dimensionality}d", + *(str(index_param.id) for index_param in index_param_prod), + f"v{format}", + ]) indexing_method = indexing_method_param.values[0] yield pytest.param( (format, dimensionality, index, indexing_method), id=id diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index b2f0c135..805704d5 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -98,19 +98,15 @@ def test_pipeline_used( @pytest.fixture def use_zarrs_direct_io() -> Generator[None]: - zarr.config.set( - { - "codec_pipeline.path": "zarrs.ZarrsCodecPipeline", - "codec_pipeline.direct_io": True, - } - ) + zarr.config.set({ + "codec_pipeline.path": "zarrs.ZarrsCodecPipeline", + "codec_pipeline.direct_io": True, + }) yield - zarr.config.set( - { - "codec_pipeline.path": "zarrs.ZarrsCodecPipeline", - "codec_pipeline.direct_io": False, - } - ) + zarr.config.set({ + "codec_pipeline.path": "zarrs.ZarrsCodecPipeline", + "codec_pipeline.direct_io": False, + }) @pytest.mark.skipif( @@ -154,12 +150,10 @@ def test_file_handle_cache(tmp_path: Path, cache_size: int) -> None: path = tmp_path / "foo.zarr" ground_truth_arr = _sharded_array(path) - with zarr.config.set( - { - "codec_pipeline.path": "zarrs.ZarrsCodecPipeline", - "codec_pipeline.file_handle_cache_size": cache_size, - } - ): + with zarr.config.set({ + "codec_pipeline.path": "zarrs.ZarrsCodecPipeline", + "codec_pipeline.file_handle_cache_size": cache_size, + }): before = _open_fds() z = zarr.open_array(path, mode="r") np.testing.assert_array_equal(z[...], ground_truth_arr) diff --git a/tests/pipeline/test_roundtrip.py b/tests/pipeline/test_roundtrip.py index db4cb4e6..a0a7ed03 100644 --- a/tests/pipeline/test_roundtrip.py +++ b/tests/pipeline/test_roundtrip.py @@ -90,9 +90,9 @@ def indexing_method(roundtrip: tuple[Literal[2, 3], int, Index, Callable]) -> Ca @contextmanager def use_zarr_default_codec_reader() -> Generator[None]: - zarr.config.set( - {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"} - ) + zarr.config.set({ + "codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline" + }) yield zarr.config.set({"codec_pipeline.path": "zarrs.ZarrsCodecPipeline"}) diff --git a/tests/test_zarrs_http.py b/tests/test_zarrs_http.py index 661f8516..45b08d76 100644 --- a/tests/test_zarrs_http.py +++ b/tests/test_zarrs_http.py @@ -6,18 +6,16 @@ import zarr from zarr.storage import FsspecStore -ARR_REF = np.array( - [ - [np.nan, np.nan, np.nan, np.nan, 0.1, 0.1, -0.6, 0.1], - [np.nan, np.nan, np.nan, np.nan, 0.1, 0.1, -1.6, 0.1], - [np.nan, np.nan, np.nan, np.nan, 0.1, 0.1, -2.6, 0.1], - [np.nan, np.nan, np.nan, np.nan, -3.4, -3.5, -3.6, 0.1], - [1.0, 1.0, 1.0, -4.3, -4.4, -4.5, -4.6, 1.1], - [1.0, 1.0, 1.0, -5.3, -5.4, -5.5, -5.6, 1.1], - [1.0, 1.0, 1.0, 1.0, 1.1, 1.1, -6.6, 1.1], - [1.0, 1.0, 1.0, 1.0, -7.4, -7.5, -7.6, -7.7], - ] -) +ARR_REF = np.array([ + [np.nan, np.nan, np.nan, np.nan, 0.1, 0.1, -0.6, 0.1], + [np.nan, np.nan, np.nan, np.nan, 0.1, 0.1, -1.6, 0.1], + [np.nan, np.nan, np.nan, np.nan, 0.1, 0.1, -2.6, 0.1], + [np.nan, np.nan, np.nan, np.nan, -3.4, -3.5, -3.6, 0.1], + [1.0, 1.0, 1.0, -4.3, -4.4, -4.5, -4.6, 1.1], + [1.0, 1.0, 1.0, -5.3, -5.4, -5.5, -5.6, 1.1], + [1.0, 1.0, 1.0, 1.0, 1.1, 1.1, -6.6, 1.1], + [1.0, 1.0, 1.0, 1.0, -7.4, -7.5, -7.6, -7.7], +]) URL = "https://raw.githubusercontent.com/zarrs/zarrs/main/zarrs/tests/data/array_write_read.zarr/group/array" From 8f377469809c74fd9b44e862f181e345ba186f2e Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Mon, 31 Aug 2026 21:47:08 +0200 Subject: [PATCH 4/4] back2crate --- src/chunk_item.rs | 2 +- src/lib.rs | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/chunk_item.rs b/src/chunk_item.rs index c7993e95..8a9aa0ad 100644 --- a/src/chunk_item.rs +++ b/src/chunk_item.rs @@ -26,7 +26,7 @@ fn to_nonzero_u64_vec(v: Vec) -> PyResult> { #[derive(Clone)] #[gen_stub_pyclass] #[pyclass] -pub struct ChunkItem { +pub(crate) struct ChunkItem { pub key: StoreKey, pub chunk_subset: ArraySubset, pub subset: ArraySubset, diff --git a/src/lib.rs b/src/lib.rs index 8424b074..9ef05f2d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,7 +43,7 @@ use crate::utils::{PyCodecErrExt, PyErrExt as _}; // TODO: Use a OnceLock for store with get_or_try_init when stabilised? #[gen_stub_pyclass] #[pyclass] -pub struct CodecPipelineImpl { +pub(crate) struct CodecPipelineImpl { pub(crate) store: ReadableWritableListableStorage, pub(crate) codec_chain: Arc, pub(crate) codec_options: CodecOptions, @@ -468,10 +468,13 @@ impl CodecPipelineImpl { /// A Python module implemented in Rust. #[pymodule] pub mod _internal { + #[pymodule_export] #[allow(non_upper_case_globals)] - pub const __version__: &str = env!("CARGO_PKG_VERSION"); - pub use super::CodecPipelineImpl; - pub use super::chunk_item::ChunkItem; + const __version__: &str = env!("CARGO_PKG_VERSION"); + #[pymodule_export] + use super::CodecPipelineImpl; + #[pymodule_export] + use super::chunk_item::ChunkItem; } define_stub_info_gatherer!(stub_info);