Add Array.prepend, bounds in the array metadata, and the offset chunk grid

changes/4516.feature.md

@@ -0,0 +1,6 @@
+Added ``Array.prepend``, which adds data before index 0 of an axis without moving existing chunks.
+The array's lower bound is read from and written to the ``bounds`` metadata member of ZEP 12
+(``inclusive_min`` / ``exclusive_max``, in place of ``shape`` once a lower bound is non-zero; later
+updates keep whichever form the document has), chunk grid indices on a ``regular`` grid can be
+negative, and ``Array.domain`` reports the coordinate range of each dimension. Added the ``offset`` chunk grid, which shifts a ``regular`` base grid so
+that its chunk 0 begins at a chosen coordinate. Chunk keys with a ``-0`` component are now rejected.

docs/user-guide/arrays.md

@@ -154,6 +154,30 @@ z.append(np.vstack([a, a]), axis=1)
 print(f"Shape after second append: {z.shape}")
 ```
 
+### Prepending
+
+[`zarr.Array.prepend`][] adds data before index 0 of an axis, the way [`zarr.Array.append`][]
+adds it after the end. The existing chunks keep their keys and their bytes. Data that does not
+fill a whole chunk goes into a partly filled first chunk, like the partly filled last chunk of
+any array.
+
+The chunk grid does not change: chunk 0 of a `regular` grid still begins at coordinate 0, and
+the chunks in front of it get negative grid indices and keys such as `c/-1/0`. What moves is the
+array's lower bound. The first prepend replaces `shape` in the stored metadata with `bounds`,
+which gives the `inclusive_min` and `exclusive_max` coordinate of every dimension, as in
+[ZEP 12](https://zarr.dev/zeps/draft/ZEP0012.html). Indexing stays zero-based, and
+[`zarr.Array.domain`][] gives the coordinate range of each dimension. Readers that do not read
+`bounds` refuse such an array rather than misread it. Later updates keep the stored form: an array
+written with `bounds` keeps `bounds` even where its lower bound is 0, and an array written with
+`shape` keeps `shape` until its first prepend.
+
+```python exec="true" session="arrays" source="above" result="ansi"
+z = zarr.create_array(store='data/example-5.zarr', shape=(6, 4), dtype='int32', chunks=(4, 4), overwrite=True)
+z[:] = 1
+z.prepend(np.full((3, 4), 2, dtype='int32'))
+print(f"Shape: {z.shape}, domain: {z.domain}")
+```
+
 ## Runtime configuration
 
 Zarr arrays are parametrized with a configuration that determines certain aspects of array behavior.

src/zarr/codecs/sharding.py

@@ -69,6 +69,7 @@
 )
 from zarr.core.metadata.v3 import (
     ChunkGridMetadata,
+    OffsetChunkGridMetadata,
     RectilinearChunkGridMetadata,
     RegularChunkGridMetadata,
     parse_codecs,
@@ -624,6 +625,8 @@ def validate(
             raise ValueError(
                 "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions."
             )
+        if isinstance(chunk_grid, OffsetChunkGridMetadata):
+            chunk_grid = chunk_grid.base_chunk_grid
         if isinstance(chunk_grid, RegularChunkGridMetadata):
             edges_per_dim: tuple[tuple[int, ...], ...] = tuple((s,) for s in chunk_grid.chunk_shape)
         elif isinstance(chunk_grid, RectilinearChunkGridMetadata):

src/zarr/core/array.py

@@ -137,6 +137,7 @@
 )
 from zarr.core.metadata.v3 import (
     ChunkGridMetadata,
+    OffsetChunkGridMetadata,
     RectilinearChunkGridMetadata,
     RegularChunkGridMetadata,
     create_chunk_grid_metadata,
@@ -263,7 +264,7 @@ def create_codec_pipeline(metadata: ArrayMetadata, *, store: Store | None = None
         # Use the regular chunk shape if available, otherwise use a
         # placeholder. The ChunkTransform is shape-agnostic — the actual
         # chunk shape is passed per-call at decode/encode time.
-        if isinstance(metadata.chunk_grid, RegularChunkGridMetadata):
+        if isinstance(metadata.chunk_grid, (RegularChunkGridMetadata, OffsetChunkGridMetadata)):
             chunk_shape = metadata.chunk_grid.chunk_shape
         else:
             chunk_shape = (1,) * len(metadata.shape)
@@ -956,6 +957,19 @@ def write_chunk_sizes(self) -> tuple[tuple[int, ...], ...]:
 
         return self._chunk_grid.chunk_sizes
 
+    @property
+    def domain(self) -> tuple[tuple[int, int], ...]:
+        """The half-open coordinate range ``(start, stop)`` of each dimension.
+
+        Index ``i`` along a dimension is coordinate ``start + i``. ``start`` is 0
+        unless the array's metadata gives ``bounds``, as it does after a `prepend`.
+        """
+        if isinstance(self.metadata, ArrayV3Metadata):
+            inclusive_min = self.metadata.inclusive_min
+        else:
+            inclusive_min = (0,) * len(self.shape)
+        return tuple((lo, lo + n) for lo, n in zip(inclusive_min, self.shape, strict=True))
+
     @property
     def shards(self) -> tuple[int, ...] | None:
         """Returns the shard shape of the Array.
@@ -1794,6 +1808,29 @@ async def append(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
         """
         return await _append(self, data, axis)
 
+    async def prepend(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
+        """Prepend `data` to `axis`.
+
+        Parameters
+        ----------
+        data : array-like
+            Data to be prepended.
+        axis : int
+            Axis along which to prepend.
+
+        Returns
+        -------
+        new_shape : tuple
+
+        Notes
+        -----
+        The size of all dimensions other than `axis` must match between this
+        array and `data`. Existing chunks keep their keys and the chunk grid is
+        unchanged; the array's lower bound moves, so its metadata carries
+        ``bounds`` in place of ``shape`` from the first prepend on.
+        """
+        return await _prepend(self, data, axis)
+
     async def update_attributes(self, new_attributes: dict[str, JSON]) -> Self:
         """
         Asynchronously update the array's attributes.
@@ -2192,6 +2229,15 @@ def write_chunk_sizes(self) -> tuple[tuple[int, ...], ...]:
         """
         return self.async_array.write_chunk_sizes
 
+    @property
+    def domain(self) -> tuple[tuple[int, int], ...]:
+        """The half-open coordinate range ``(start, stop)`` of each dimension.
+
+        Index ``i`` along a dimension is coordinate ``start + i``. ``start`` is 0
+        unless the array's metadata gives ``bounds``, as it does after a `prepend`.
+        """
+        return self.async_array.domain
+
     @property
     def shards(self) -> tuple[int, ...] | None:
         """Returns a tuple of integers describing the length of each dimension of a shard of the array.
@@ -4011,6 +4057,29 @@ def append(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
         """
         return sync(self.async_array.append(data, axis=axis))
 
+    def prepend(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
+        """Prepend `data` to `axis`.
+
+        Parameters
+        ----------
+        data : array-like
+            Data to be prepended.
+        axis : int
+            Axis along which to prepend.
+
+        Returns
+        -------
+        new_shape : tuple
+
+        Notes
+        -----
+        The size of all dimensions other than `axis` must match between this
+        array and `data`. Existing chunks keep their keys and the chunk grid is
+        unchanged; the array's lower bound moves, so its metadata carries
+        ``bounds`` in place of ``shape`` from the first prepend on.
+        """
+        return sync(self.async_array.prepend(data, axis=axis))
+
     def update_attributes(self, new_attributes: dict[str, JSON]) -> Self:
         """
         Update the array's attributes.
@@ -5558,7 +5627,7 @@ async def _nchunks_initialized(
     grid = array._chunk_grid
     total = 0
     for key in await _shards_initialized(array):
-        spec = grid[meta.chunk_key_encoding.decode_chunk_key(key)]
+        spec = grid[meta.decode_chunk_key(key)]
         if spec is not None:
             total += product(
                 tuple(s // c for s, c in zip(spec.codec_shape, inner_chunks, strict=True))
@@ -6059,6 +6128,69 @@ async def _append(
     return new_shape
 
 
+async def _prepend(
+    array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata],
+    data: npt.ArrayLike,
+    axis: int = 0,
+) -> tuple[int, ...]:
+    """
+    Prepend data to an array along the specified axis.
+
+    Parameters
+    ----------
+    array : AsyncArray
+        The array to prepend to. Must be a Zarr v3 array with a regular chunk grid.
+    data : npt.ArrayLike
+        Data to be prepended.
+    axis : int
+        Axis along which to prepend.
+
+    Returns
+    -------
+    new_shape : tuple[int, ...]
+        The new shape of the array after prepending.
+    """
+    if not hasattr(data, "shape"):
+        data = np.asanyarray(data)
+
+    metadata = array.metadata
+    if not isinstance(metadata, ArrayV3Metadata) or not isinstance(
+        metadata.chunk_grid, (RegularChunkGridMetadata, OffsetChunkGridMetadata)
+    ):
+        raise TypeError("prepend needs a Zarr v3 array with a regular chunk grid.")
+    ndim = len(metadata.shape)
+    if not -ndim <= axis < ndim:
+        raise ValueError(f"axis {axis} is out of range for a {ndim}-dimensional array.")
+    axis %= ndim
+
+    self_shape_preserved = tuple(s for i, s in enumerate(array.shape) if i != axis)
+    data_shape_preserved = tuple(s for i, s in enumerate(data.shape) if i != axis)
+    if self_shape_preserved != data_shape_preserved:
+        raise ValueError(
+            f"shape of data to prepend is not compatible with the array. "
+            f"The shape of the data is ({data_shape_preserved})"
+            f"and the shape of the array is ({self_shape_preserved})."
+            "All dimensions must match except for the dimension being "
+            "prepended."
+        )
+    extent = data.shape[axis]
+    new_shape = tuple(s + extent if i == axis else s for i, s in enumerate(metadata.shape))
+    new_inclusive_min = tuple(
+        lo - extent if i == axis else lo for i, lo in enumerate(metadata.inclusive_min)
+    )
+    new_metadata = replace(metadata, shape=new_shape, inclusive_min=new_inclusive_min)
+    new_chunk_grid = ChunkGrid.from_metadata(new_metadata)
+    await store_documents(array.store_path, encode_documents(array.store_path, new_metadata))
+    array._stored_document_replaced()
+    object.__setattr__(array, "metadata", new_metadata)
+    object.__setattr__(array, "_chunk_grid", new_chunk_grid)
+
+    prepend_selection = tuple(slice(None) if i != axis else slice(0, extent) for i in range(ndim))
+    await array.setitem(prepend_selection, data)
+
+    return new_shape
+
+
 async def _update_attributes(
     array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata],
     new_attributes: dict[str, JSON],

src/zarr/core/chunk_grids.py

@@ -48,10 +48,12 @@
 @dataclass(frozen=True)
 class FixedDimension:
     """Uniform chunk size. Boundary chunks contain less data but are
-    encoded at full size by the codec pipeline."""
+    encoded at full size by the codec pipeline. With a non-zero ``shift`` the
+    first element sits ``shift`` positions into the first chunk's buffer."""
 
     size: int  # chunk edge length (>= 1)
     extent: int  # array dimension length (>= 0)
+    shift: int = 0  # position of array index 0 inside the first chunk, in [0, size)
     nchunks: int = field(init=False, repr=False)
     ngridcells: int = field(init=False, repr=False)
 
@@ -60,7 +62,9 @@ def __post_init__(self) -> None:
             raise ValueError(f"FixedDimension size must be >= 1, got {self.size}")
         if self.extent < 0:
             raise ValueError(f"FixedDimension extent must be >= 0, got {self.extent}")
-        n = ceildiv_int(self.extent, self.size)
+        if not 0 <= self.shift < self.size:
+            raise ValueError(f"FixedDimension shift must be in [0, {self.size}), got {self.shift}")
+        n = ceildiv_int(self.extent + self.shift, self.size) if self.extent else 0
         object.__setattr__(self, "nchunks", n)
         object.__setattr__(self, "ngridcells", n)
 
@@ -69,15 +73,19 @@ def index_to_chunk(self, idx: int) -> int:
             raise IndexError(f"Negative index {idx} is not allowed")
         if idx >= self.extent:
             raise IndexError(f"Index {idx} is out of bounds for extent {self.extent}")
-        return idx // self.size
+        return (idx + self.shift) // self.size
 
     def chunk_offset(self, chunk_ix: int) -> int:
-        """Byte-aligned start position of chunk *chunk_ix* in array coordinates.
+        """Start of the valid data region of chunk *chunk_ix* in array coordinates.
 
         Does not validate *chunk_ix* — callers must ensure it is in
         ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access.
         """
-        return chunk_ix * self.size
+        return max(0, chunk_ix * self.size - self.shift)
+
+    def buffer_offset(self, chunk_ix: int) -> int:
+        """Position of the valid data region inside the chunk's buffer."""
+        return self.shift if chunk_ix == 0 else 0
 
     def chunk_size(self, chunk_ix: int) -> int:
         """Buffer size for codec processing — always uniform.
@@ -93,7 +101,8 @@ def data_size(self, chunk_ix: int) -> int:
         Does not validate *chunk_ix* — callers must ensure it is in
         ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access.
         """
-        return max(0, min(self.size, self.extent - chunk_ix * self.size))
+        start = self.chunk_offset(chunk_ix)
+        return max(0, min((chunk_ix + 1) * self.size - self.shift, self.extent) - start)
 
     @property
     def _unique_edge_lengths(self) -> Iterable[int]:
@@ -106,7 +115,7 @@ def _unique_edge_lengths(self) -> Iterable[int]:
         return (self.size,)
 
     def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]:
-        return indices // self.size
+        return (indices + self.shift) // self.size
 
     def with_extent(self, new_extent: int) -> FixedDimension:
         """Re-bind to *new_extent* without modifying edges.
@@ -115,15 +124,15 @@ def with_extent(self, new_extent: int) -> FixedDimension:
         are already correct. Raises on
         ``VaryingDimension`` if edges don't cover the new extent.
         """
-        return FixedDimension(size=self.size, extent=new_extent)
+        return FixedDimension(size=self.size, extent=new_extent, shift=self.shift)
 
     def resize(self, new_extent: int) -> FixedDimension:
         """Adapt for a user-initiated array resize, growing edges if needed.
 
         For ``FixedDimension`` this is identical to ``with_extent`` since
         regular grids don't store explicit edges.
         """
-        return FixedDimension(size=self.size, extent=new_extent)
+        return FixedDimension(size=self.size, extent=new_extent, shift=self.shift)
 
     @property
     def _size_repr(self) -> str:
@@ -205,6 +214,10 @@ def data_size(self, chunk_ix: int) -> int:
         offset = self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0
         return max(0, min(self.edges[chunk_ix], self.extent - offset))
 
+    def buffer_offset(self, chunk_ix: int) -> int:
+        """Position of the valid data region inside the chunk's buffer."""
+        return 0
+
     @property
     def _unique_edge_lengths(self) -> Iterable[int]:
         """Distinct chunk edge lengths for this dimension (lazily deduplicated).
@@ -271,6 +284,7 @@ def index_to_chunk(self, idx: int) -> int: ...
     def chunk_offset(self, chunk_ix: int) -> int: ...
     def chunk_size(self, chunk_ix: int) -> int: ...
     def data_size(self, chunk_ix: int) -> int: ...
+    def buffer_offset(self, chunk_ix: int) -> int: ...
     def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: ...
     @property
     def _unique_edge_lengths(self) -> Iterable[int]: ...
@@ -416,13 +430,27 @@ def from_metadata(cls, metadata: ArrayMetadata) -> ChunkGrid:
         For v3 metadata, dispatches on the chunk grid type.
         """
         from zarr.core.metadata import ArrayV2Metadata
-        from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata
+        from zarr.core.metadata.v3 import (
+            OffsetChunkGridMetadata,
+            RectilinearChunkGridMetadata,
+            RegularChunkGridMetadata,
+        )
 
         if isinstance(metadata, ArrayV2Metadata):
             return cls.from_sizes(metadata.shape, tuple(metadata.chunks))
         chunk_grid_meta = metadata.chunk_grid
-        if isinstance(chunk_grid_meta, RegularChunkGridMetadata):
-            return cls.from_sizes(metadata.shape, tuple(chunk_grid_meta.chunk_shape))
+        if isinstance(chunk_grid_meta, (RegularChunkGridMetadata, OffsetChunkGridMetadata)):
+            return cls(
+                dimensions=tuple(
+                    FixedDimension(size=c, extent=e, shift=s)
+                    for c, e, s in zip(
+                        chunk_grid_meta.chunk_shape,
+                        metadata.shape,
+                        metadata.first_chunk_offset,
+                        strict=True,
+                    )
+                )
+            )
         elif isinstance(chunk_grid_meta, RectilinearChunkGridMetadata):
             return cls.from_sizes(metadata.shape, chunk_grid_meta.chunk_shapes)
         else:
@@ -815,8 +843,16 @@ def normalize_chunks_nd(
     chunk shape, when normalizing a shard shape); it only affects the chunks
     that `-1` and `False` derive from the span.
     """
-    from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata
+    from zarr.core.metadata.v3 import (
+        OffsetChunkGridMetadata,
+        RectilinearChunkGridMetadata,
+        RegularChunkGridMetadata,
+    )
 
+    if isinstance(chunks, OffsetChunkGridMetadata):
+        raise TypeError(
+            "An offset chunk grid cannot be passed as `chunks`; pass its base grid's chunk shape."
+        )
     if isinstance(chunks, RegularChunkGridMetadata):
         return ChunkGrid.from_sizes(shape, tuple(chunks.chunk_shape))
     if isinstance(chunks, RectilinearChunkGridMetadata):