cpp-zarr

pip install cpp-zarr

import cppzarr
im = cppzarr.read_zarr('filename.zarr')
# Do some processing here
cppzarr.write_zarr('outputFilename.zarr', im)

# Write a Zarr v3 file instead
cppzarr.write_zarr('outputFilename.zarr', im, zarr_format=3)

import cppzarr

# Optionally specify the start coordinates
im = cppzarr.read_zarr('filename.zarr', start_coords=[10, 10, 10])

# Optionally specify the end coordinates
im1 = cppzarr.read_zarr('filename.zarr', end_coords=[100, 100, 100])

# Optionally specify the start and end coordinates
im2 = cppzarr.read_zarr('filename.zarr', start_coords=[10, 10, 10], end_coords=[100, 100, 100])

import cppzarr
im = cppzarr.read_zarr('filename.zarr', order='C')

import cppzarr
im = cppzarr.read_zarr('filename.zarr')

# Optionally specify specific metadata
cppzarr.write_zarr('outputFilename.zarr', im, cname='zstd', clevel=1, order='F', chunks=[256, 256, 256], dimension_separator='/')

# New Zarr v2 files put their chunk files in nested folders (dimension_separator='/', files named 0/0/0) by default.
# dimension_separator='.' names them 0.0.0 in one folder, which writes much slower on network file systems.

# Zarr v3 with 1024x1024x1024 shards made of 256x256x256 chunks (shards must be a multiple of chunks).
# Zarr v3 files default to order='C' and dimension_separator='/' (chunk files named c/0/0/0)
cppzarr.write_zarr('outputFilename.zarr', im, zarr_format=3, cname='zstd', clevel=1, chunks=[256, 256, 256], shards=[1024, 1024, 1024])

import numpy as np
import cppzarr

# The array's data type is the numpy array's: float16, complex64 and bool arrays included
cppzarr.write_zarr('half.zarr', np.random.rand(256, 256, 64).astype(np.float16))

# bfloat16 arrays are ml_dtypes' bfloat16 type
import ml_dtypes
cppzarr.write_zarr('bf16.zarr', np.random.rand(256, 256, 64).astype(ml_dtypes.bfloat16), zarr_format=3)

import numpy as np
import cppzarr

# A call uses every CPU by default (or OMP_NUM_THREADS); num_threads sets it for one call
im = cppzarr.read_zarr('filename.zarr', num_threads=4)

# Read a region into an existing array instead of a new one (of the region's shape and the
# array's data type, C- or F-contiguous), such as one slot of a batch buffer
batch = np.empty((8, 128, 128, 128), dtype=np.uint16)
cppzarr.read_zarr('filename.zarr', [0, 0, 0], [128, 128, 128], out=batch[0])

# Reads and writes release the GIL, so several Python threads can read at once: give each
# a share of the CPUs with num_threads. The same goes for several processes reading at once
# (a multiprocessing pool, a data loader's workers): give each process a share with
# num_threads (a bigger one to those with more to read), or OMP_NUM_THREADS set before
# cppzarr is imported. Processes that each use every CPU make one another's reads several
# times slower.

# Or read a batch of regions at once (from one array or several), new arrays or into given ones:
# the reads overlap and share num_threads threads (by default, as for a call, OMP_NUM_THREADS
# or every CPU), each read getting more of them the larger it is
regions = [('filename.zarr', [0, 0, 128*i], [128, 128, 128*(i+1)]) for i in range(8)]
arrays = cppzarr.read_zarr_batch(regions, num_threads=8)
cppzarr.read_zarr_batch([r + (batch[i],) for i, r in enumerate(regions)], num_threads=8)

import cppzarr
im = cppzarr.read_zarr('filename.zarr')

# Write the zarr file out normally
cppzarr.write_zarr('filename.zarr', im)

# Write to a specified region with different data (the file keeps its Zarr version)
cppzarr.write_zarr('filename.zarr', im[100:200,100:200,100:200], start_coords=[0,0,0], end_coords=[100,100,100])

import cppzarr

# Only the metadata is rewritten (.zarray and .zattrs become zarr.json); the chunk files are kept.
# Files with subfolders or numcodecs zlib compression cannot be converted
cppzarr.convert_to_v3('filename.zarr')

git clone https://github.com/abcucberkeley/cpp-zarr
cd cpp-zarr
mkdir build
cd build
cmake ..
make -j
make install

% Note the created .zarray file is probably hidden by default on your system
createZarrFile('path/to/file.zarr');
% Zarr v3 metadata (zarr.json) instead, with 1024x1024x1024 shards made of 256x256x256 chunks
createZarrFile('path/to/file.zarr','shape',[2048 2048 1024],'dtype','<u2','chunks',[1024 1024 1024],'chunk_shape',[256 256 256],'zarr_format',3);

im = parallelReadZarr('path/to/file.zarr');
% Read a region: [starts ends] with one start and one end per axis (1-based, inclusive)
im = parallelReadZarr('path/to/file.zarr', 'bbox', [1 1 1 100 100 50]);

im = rand(100,100,100);
parallelWriteZarr('path/to/file.zarr',im);
% Write with a chunk size (one value per axis)
parallelWriteZarr('path/to/file.zarr',im,'chunks',[64 64 64]);
% Write with a compressor and compression level (blosc's lz4 at level 5 by default; levels go from 0 to 9)
parallelWriteZarr('path/to/file.zarr',im,'cname','zstd','clevel',1);
% Write a new array of another data type than the data's (here float16; also 'bfloat16', 'uint16', 'single', '<u2', ...).
% MATLAB has no 16-bit float class: float16 and bfloat16 arrays read as single. Logical arrays are written as bool,
% complex single and double as complex64 and complex128, and data written into an existing array is converted to its
% data type as MATLAB converts between classes.
parallelWriteZarr('path/to/file.zarr',single(im),'dtype','float16');
% Write into a region of an existing Zarr file ([starts ends] as in parallelReadZarr); the file keeps its Zarr version
parallelWriteZarr('path/to/file.zarr',im(1:50,:,:),'bbox',[1 1 1 50 100 100]);
% Write a Zarr v3 file (C order and chunk files named c/0/0/0 by default; 'order','F' still works)
parallelWriteZarr('path/to/file.zarr',im,'zarr_format',3);
% New Zarr v2 files put their chunk files in nested folders (files named 0/0/0) by default. '.' names them
% 0.0.0 in one folder, which writes much slower on network file systems.
parallelWriteZarr('path/to/file.zarr',im,'dimension_separator','.');

% Only the metadata is rewritten (.zarray and .zattrs become zarr.json); the chunk files are kept.
% Files with subfolders or numcodecs zlib compression cannot be converted
convertZarrToV3('path/to/file.zarr');