@az0uz/zarr
Version:
Javascript implementation of Zarr
1,419 lines (1,407 loc) • 133 kB
JavaScript
const registry = new Map();
function addCodec(id, importFn) {
registry.set(id, importFn);
}
async function getCodec(config) {
if (!registry.has(config.id)) {
throw new Error(`Compression codec ${config.id} is not supported by Zarr.js yet.`);
}
/* eslint-disable @typescript-eslint/no-non-null-assertion */
const codec = await registry.get(config.id)();
return codec.fromConfig(config);
}
function createProxy(mapping) {
return new Proxy(mapping, {
set(target, key, value, _receiver) {
return target.setItem(key, value);
},
get(target, key, _receiver) {
return target.getItem(key);
},
deleteProperty(target, key) {
return target.deleteItem(key);
},
has(target, key) {
return target.containsItem(key);
}
});
}
function isZarrError(err) {
return typeof err === 'object' && err !== null && '__zarr__' in err;
}
function isKeyError(o) {
return isZarrError(o) && o.__zarr__ === 'KeyError';
}
// Custom error messages, note we have to patch the prototype of the
// errors to fix `instanceof` calls, see:
// https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
class ContainsArrayError extends Error {
constructor(path) {
super(`path ${path} contains an array`);
this.__zarr__ = 'ContainsArrayError';
Object.setPrototypeOf(this, ContainsArrayError.prototype);
}
}
class ContainsGroupError extends Error {
constructor(path) {
super(`path ${path} contains a group`);
this.__zarr__ = 'ContainsGroupError';
Object.setPrototypeOf(this, ContainsGroupError.prototype);
}
}
class ArrayNotFoundError extends Error {
constructor(path) {
super(`array not found at path ${path}`);
this.__zarr__ = 'ArrayNotFoundError';
Object.setPrototypeOf(this, ArrayNotFoundError.prototype);
}
}
class GroupNotFoundError extends Error {
constructor(path) {
super(`group not found at path ${path}`);
this.__zarr__ = 'GroupNotFoundError';
Object.setPrototypeOf(this, GroupNotFoundError.prototype);
}
}
class PathNotFoundError extends Error {
constructor(path) {
super(`nothing found at path ${path}`);
this.__zarr__ = 'PathNotFoundError';
Object.setPrototypeOf(this, PathNotFoundError.prototype);
}
}
class PermissionError extends Error {
constructor(message) {
super(message);
this.__zarr__ = 'PermissionError';
Object.setPrototypeOf(this, PermissionError.prototype);
}
}
class KeyError extends Error {
constructor(key) {
super(`key ${key} not present`);
this.__zarr__ = 'KeyError';
Object.setPrototypeOf(this, KeyError.prototype);
}
}
class TooManyIndicesError extends RangeError {
constructor(selection, shape) {
super(`too many indices for array; expected ${shape.length}, got ${selection.length}`);
this.__zarr__ = 'TooManyIndicesError';
Object.setPrototypeOf(this, TooManyIndicesError.prototype);
}
}
class BoundsCheckError extends RangeError {
constructor(message) {
super(message);
this.__zarr__ = 'BoundsCheckError';
Object.setPrototypeOf(this, BoundsCheckError.prototype);
}
}
class InvalidSliceError extends RangeError {
constructor(from, to, stepSize, reason) {
super(`slice arguments slice(${from}, ${to}, ${stepSize}) invalid: ${reason}`);
this.__zarr__ = 'InvalidSliceError';
Object.setPrototypeOf(this, InvalidSliceError.prototype);
}
}
class NegativeStepError extends Error {
constructor() {
super(`Negative step size is not supported when indexing.`);
this.__zarr__ = 'NegativeStepError';
Object.setPrototypeOf(this, NegativeStepError.prototype);
}
}
class ValueError extends Error {
constructor(message) {
super(message);
this.__zarr__ = 'ValueError';
Object.setPrototypeOf(this, ValueError.prototype);
}
}
class HTTPError extends Error {
constructor(code) {
super(code);
this.__zarr__ = 'HTTPError';
Object.setPrototypeOf(this, HTTPError.prototype);
}
}
function slice(start, stop = undefined, step = null) {
// tslint:disable-next-line: strict-type-predicates
if (start === undefined) { // Not possible in typescript
throw new InvalidSliceError(start, stop, step, "The first argument must not be undefined");
}
if ((typeof start === "string" && start !== ":") || (typeof stop === "string" && stop !== ":")) { // Note in typescript this will never happen with type checking.
throw new InvalidSliceError(start, stop, step, "Arguments can only be integers, \":\" or null");
}
// slice(5) === slice(null, 5)
if (stop === undefined) {
stop = start;
start = null;
}
// if (start !== null && stop !== null && start > stop) {
// throw new InvalidSliceError(start, stop, step, "to is higher than from");
// }
return {
start: start === ":" ? null : start,
stop: stop === ":" ? null : stop,
step,
_slice: true,
};
}
/**
* Port of adjustIndices
* https://github.com/python/cpython/blob/master/Objects/sliceobject.c#L243
*/
function adjustIndices(start, stop, step, length) {
if (start < 0) {
start += length;
if (start < 0) {
start = (step < 0) ? -1 : 0;
}
}
else if (start >= length) {
start = (step < 0) ? length - 1 : length;
}
if (stop < 0) {
stop += length;
if (stop < 0) {
stop = (step < 0) ? -1 : 0;
}
}
else if (stop >= length) {
stop = (step < 0) ? length - 1 : length;
}
if (step < 0) {
if (stop < start) {
const length = Math.floor((start - stop - 1) / (-step) + 1);
return [start, stop, step, length];
}
}
else {
if (start < stop) {
const length = Math.floor((stop - start - 1) / step + 1);
return [start, stop, step, length];
}
}
return [start, stop, step, 0];
}
/**
* Port of slice.indices(n) and PySlice_Unpack
* https://github.com/python/cpython/blob/master/Objects/sliceobject.c#L166
* https://github.com/python/cpython/blob/master/Objects/sliceobject.c#L198
*
* Behaviour might be slightly different as it's a weird hybrid implementation.
*/
function sliceIndices(slice, length) {
let start;
let stop;
let step;
if (slice.step === null) {
step = 1;
}
else {
step = slice.step;
}
if (slice.start === null) {
start = step < 0 ? Number.MAX_SAFE_INTEGER : 0;
}
else {
start = slice.start;
if (start < 0) {
start += length;
}
}
if (slice.stop === null) {
stop = step < 0 ? -Number.MAX_SAFE_INTEGER : Number.MAX_SAFE_INTEGER;
}
else {
stop = slice.stop;
if (stop < 0) {
stop += length;
}
}
// This clips out of bounds slices
const s = adjustIndices(start, stop, step, length);
start = s[0];
stop = s[1];
step = s[2];
// The output length
length = s[3];
// With out of bounds slicing these two assertions are not useful.
// if (stop > length) throw new Error("Stop greater than length");
// if (start >= length) throw new Error("Start greater than or equal to length");
if (step === 0)
throw new Error("Step size 0 is invalid");
return [start, stop, step, length];
}
function ensureArray(selection) {
if (!Array.isArray(selection)) {
return [selection];
}
return selection;
}
function checkSelectionLength(selection, shape) {
if (selection.length > shape.length) {
throw new TooManyIndicesError(selection, shape);
}
}
/**
* Returns both the sliceIndices per dimension and the output shape after slicing.
*/
function selectionToSliceIndices(selection, shape) {
const sliceIndicesResult = [];
const outShape = [];
for (let i = 0; i < selection.length; i++) {
const s = selection[i];
if (typeof s === "number") {
sliceIndicesResult.push(s);
}
else {
const x = sliceIndices(s, shape[i]);
const dimLength = x[3];
outShape.push(dimLength);
sliceIndicesResult.push(x);
}
}
return [sliceIndicesResult, outShape];
}
/**
* This translates "...", ":", null into a list of slices or non-negative integer selections of length shape
*/
function normalizeArraySelection(selection, shape, convertIntegerSelectionToSlices = false) {
selection = replaceEllipsis(selection, shape);
for (let i = 0; i < selection.length; i++) {
const dimSelection = selection[i];
if (typeof dimSelection === "number") {
if (convertIntegerSelectionToSlices) {
selection[i] = slice(dimSelection, dimSelection + 1, 1);
}
else {
selection[i] = normalizeIntegerSelection(dimSelection, shape[i]);
}
}
else if (isIntegerArray(dimSelection)) {
throw new TypeError("Integer array selections are not supported (yet)");
}
else if (dimSelection === ":" || dimSelection === null) {
selection[i] = slice(null, null, 1);
}
}
return selection;
}
function replaceEllipsis(selection, shape) {
selection = ensureArray(selection);
let ellipsisIndex = -1;
let numEllipsis = 0;
for (let i = 0; i < selection.length; i++) {
if (selection[i] === "...") {
ellipsisIndex = i;
numEllipsis += 1;
}
}
if (numEllipsis > 1) {
throw new RangeError("an index can only have a single ellipsis ('...')");
}
if (numEllipsis === 1) {
// count how many items to left and right of ellipsis
const numItemsLeft = ellipsisIndex;
const numItemsRight = selection.length - (numItemsLeft + 1);
const numItems = selection.length - 1; // All non-ellipsis items
if (numItems >= shape.length) {
// Ellipsis does nothing, just remove it
selection = selection.filter((x) => x !== "...");
}
else {
// Replace ellipsis with as many slices are needed for number of dims
const numNewItems = shape.length - numItems;
let newItem = selection.slice(0, numItemsLeft).concat(new Array(numNewItems).fill(null));
if (numItemsRight > 0) {
newItem = newItem.concat(selection.slice(selection.length - numItemsRight));
}
selection = newItem;
}
}
// Fill out selection if not completely specified
if (selection.length < shape.length) {
const numMissing = shape.length - selection.length;
selection = selection.concat(new Array(numMissing).fill(null));
}
checkSelectionLength(selection, shape);
return selection;
}
function normalizeIntegerSelection(dimSelection, dimLength) {
// Note: Maybe we should convert to integer or warn if dimSelection is not an integer
// handle wraparound
if (dimSelection < 0) {
dimSelection = dimLength + dimSelection;
}
// handle out of bounds
if (dimSelection >= dimLength || dimSelection < 0) {
throw new BoundsCheckError(`index out of bounds for dimension with length ${dimLength}`);
}
return dimSelection;
}
function isInteger(s) {
return typeof s === "number";
}
function isIntegerArray(s) {
if (!Array.isArray(s)) {
return false;
}
for (const e of s) {
if (typeof e !== "number") {
return false;
}
}
return true;
}
function isSlice(s) {
if (s !== null && s["_slice"] === true) {
return true;
}
return false;
}
function isContiguousSlice(s) {
return isSlice(s) && (s.step === null || s.step === 1);
}
function isContiguousSelection(selection) {
selection = ensureArray(selection);
for (let i = 0; i < selection.length; i++) {
const s = selection[i];
if (!(isIntegerArray(s) || isContiguousSlice(s) || s === "...")) {
return false;
}
}
return true;
}
function* product(...iterables) {
if (iterables.length === 0) {
return;
}
// make a list of iterators from the iterables
const iterators = iterables.map(it => it());
const results = iterators.map(it => it.next());
// Disabled to allow empty inputs
// if (results.some(r => r.done)) {
// throw new Error("Input contains an empty iterator.");
// }
for (let i = 0;;) {
if (results[i].done) {
// reset the current iterator
iterators[i] = iterables[i]();
results[i] = iterators[i].next();
// advance, and exit if we've reached the end
if (++i >= iterators.length) {
return;
}
}
else {
yield results.map(({ value }) => value);
i = 0;
}
results[i] = iterators[i].next();
}
}
class BasicIndexer {
constructor(selection, array) {
selection = normalizeArraySelection(selection, array.shape);
// Setup per-dimension indexers
this.dimIndexers = [];
const arrayShape = array.shape;
for (let i = 0; i < arrayShape.length; i++) {
let dimSelection = selection[i];
const dimLength = arrayShape[i];
const dimChunkLength = array.chunks[i];
if (dimSelection === null) {
dimSelection = slice(null);
}
if (isInteger(dimSelection)) {
this.dimIndexers.push(new IntDimIndexer(dimSelection, dimLength, dimChunkLength));
}
else if (isSlice(dimSelection)) {
this.dimIndexers.push(new SliceDimIndexer(dimSelection, dimLength, dimChunkLength));
}
else {
throw new RangeError(`Unspported selection item for basic indexing; expected integer or slice, got ${dimSelection}`);
}
}
this.shape = [];
for (const d of this.dimIndexers) {
if (d instanceof SliceDimIndexer) {
this.shape.push(d.numItems);
}
}
this.dropAxes = null;
}
*iter() {
const dimIndexerIterables = this.dimIndexers.map(x => (() => x.iter()));
const dimIndexerProduct = product(...dimIndexerIterables);
for (const dimProjections of dimIndexerProduct) {
// TODO fix this, I think the product outputs too many combinations
const chunkCoords = [];
const chunkSelection = [];
const outSelection = [];
for (const p of dimProjections) {
chunkCoords.push((p).dimChunkIndex);
chunkSelection.push((p).dimChunkSelection);
if ((p).dimOutSelection !== null) {
outSelection.push((p).dimOutSelection);
}
}
yield {
chunkCoords,
chunkSelection,
outSelection,
};
}
}
}
class IntDimIndexer {
constructor(dimSelection, dimLength, dimChunkLength) {
dimSelection = normalizeIntegerSelection(dimSelection, dimLength);
this.dimSelection = dimSelection;
this.dimLength = dimLength;
this.dimChunkLength = dimChunkLength;
this.numItems = 1;
}
*iter() {
const dimChunkIndex = Math.floor(this.dimSelection / this.dimChunkLength);
const dimOffset = dimChunkIndex * this.dimChunkLength;
const dimChunkSelection = this.dimSelection - dimOffset;
const dimOutSelection = null;
yield {
dimChunkIndex,
dimChunkSelection,
dimOutSelection,
};
}
}
class SliceDimIndexer {
constructor(dimSelection, dimLength, dimChunkLength) {
// Normalize
const [start, stop, step] = sliceIndices(dimSelection, dimLength);
this.start = start;
this.stop = stop;
this.step = step;
if (this.step < 1) {
throw new NegativeStepError();
}
this.dimLength = dimLength;
this.dimChunkLength = dimChunkLength;
this.numItems = Math.max(0, Math.ceil((this.stop - this.start) / this.step));
this.numChunks = Math.ceil(this.dimLength / this.dimChunkLength);
}
*iter() {
const dimChunkIndexFrom = Math.floor(this.start / this.dimChunkLength);
const dimChunkIndexTo = Math.ceil(this.stop / this.dimChunkLength);
// Iterate over chunks in range
for (let dimChunkIndex = dimChunkIndexFrom; dimChunkIndex < dimChunkIndexTo; dimChunkIndex++) {
// Compute offsets for chunk within overall array
const dimOffset = dimChunkIndex * this.dimChunkLength;
const dimLimit = Math.min(this.dimLength, (dimChunkIndex + 1) * this.dimChunkLength);
// Determine chunk length, accounting for trailing chunk
const dimChunkLength = dimLimit - dimOffset;
let dimChunkSelStart;
let dimChunkSelStop;
let dimOutOffset;
if (this.start < dimOffset) {
// Selection starts before current chunk
dimChunkSelStart = 0;
const remainder = (dimOffset - this.start) % this.step;
if (remainder > 0) {
dimChunkSelStart += this.step - remainder;
}
// Compute number of previous items, provides offset into output array
dimOutOffset = Math.ceil((dimOffset - this.start) / this.step);
}
else {
// Selection starts within current chunk
dimChunkSelStart = this.start - dimOffset;
dimOutOffset = 0;
}
if (this.stop > dimLimit) {
// Selection ends after current chunk
dimChunkSelStop = dimChunkLength;
}
else {
// Selection ends within current chunk
dimChunkSelStop = this.stop - dimOffset;
}
const dimChunkSelection = slice(dimChunkSelStart, dimChunkSelStop, this.step);
const dimChunkNumItems = Math.ceil((dimChunkSelStop - dimChunkSelStart) / this.step);
const dimOutSelection = slice(dimOutOffset, dimOutOffset + dimChunkNumItems);
yield {
dimChunkIndex,
dimChunkSelection,
dimOutSelection,
};
}
}
}
/**
* This should be true only if this javascript is getting executed in Node.
*/
const IS_NODE = typeof process !== "undefined" && process.versions && process.versions.node;
// eslint-disable-next-line @typescript-eslint/no-empty-function
function noop() { }
// eslint-disable-next-line @typescript-eslint/ban-types
function normalizeStoragePath(path) {
if (path === null) {
return "";
}
if (path instanceof String) {
path = path.valueOf();
}
// convert backslash to forward slash
path = path.replace(/\\/g, "/");
// ensure no leading slash
while (path.length > 0 && path[0] === '/') {
path = path.slice(1);
}
// ensure no trailing slash
while (path.length > 0 && path[path.length - 1] === '/') {
path = path.slice(0, path.length - 1);
}
// collapse any repeated slashes
path = path.replace(/\/\/+/g, "/");
// don't allow path segments with just '.' or '..'
const segments = path.split('/');
for (const s of segments) {
if (s === "." || s === "..") {
throw Error("path containing '.' or '..' segment not allowed");
}
}
return path;
}
function normalizeShape(shape) {
if (typeof shape === "number") {
shape = [shape];
}
return shape.map(x => Math.floor(x));
}
function normalizeChunks(chunks, shape) {
// Assume shape is already normalized
if (chunks === null || chunks === true) {
throw new Error("Chunk guessing is not supported yet");
}
if (chunks === false) {
return shape;
}
if (typeof chunks === "number") {
chunks = [chunks];
}
// handle underspecified chunks
if (chunks.length < shape.length) {
// assume chunks across remaining dimensions
chunks = chunks.concat(shape.slice(chunks.length));
}
return chunks.map((x, idx) => {
// handle null or -1 in chunks
if (x === -1 || x === null) {
return shape[idx];
}
else {
return Math.floor(x);
}
});
}
function normalizeOrder(order) {
order = order.toUpperCase();
return order;
}
function normalizeDtype(dtype) {
return dtype;
}
function normalizeFillValue(fillValue) {
return fillValue;
}
/**
* Determine whether `item` specifies a complete slice of array with the
* given `shape`. Used to optimize __setitem__ operations on chunks
* @param item
* @param shape
*/
function isTotalSlice(item, shape) {
if (item === null) {
return true;
}
if (!Array.isArray(item)) {
item = [item];
}
for (let i = 0; i < Math.min(item.length, shape.length); i++) {
const it = item[i];
if (it === null)
continue;
if (isSlice(it)) {
const s = it;
const isStepOne = s.step === 1 || s.step === null;
if (s.start === null && s.stop === null && isStepOne) {
continue;
}
if ((s.stop - s.start) === shape[i] && isStepOne) {
continue;
}
return false;
}
return false;
// } else {
// console.error(`isTotalSlice unexpected non-slice, got ${it}`);
// return false;
// }
}
return true;
}
/**
* Checks for === equality of all elements.
*/
function arrayEquals1D(a, b) {
if (a.length !== b.length) {
return false;
}
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) {
return false;
}
}
return true;
}
/*
* Determines "C" order strides for a given shape array.
* Strides provide integer steps in each dimention to traverse an ndarray.
*
* NOTE: - These strides here are distinct from numpy.ndarray.strides, which describe actual byte steps.
* - Strides are assumed to be contiguous, so initial step is 1. Thus, output will always be [XX, XX, 1].
*/
function getStrides(shape) {
// adapted from https://github.com/scijs/ndarray/blob/master/ndarray.js#L326-L330
const ndim = shape.length;
const strides = Array(ndim);
let step = 1; // init step
for (let i = ndim - 1; i >= 0; i--) {
strides[i] = step;
step *= shape[i];
}
return strides;
}
function resolveUrl(root, path) {
const base = typeof root === 'string' ? new URL(root) : root;
if (!base.pathname.endsWith('/')) {
// ensure trailing slash so that base is resolved as _directory_
base.pathname += '/';
}
const resolved = new URL(path, base);
// copy search params to new URL
resolved.search = base.search;
return resolved.href;
}
/**
* Swaps byte order in-place for a given TypedArray.
* Used to flip endian-ness when getting/setting chunks from/to zarr store.
* @param src TypedArray
*/
function byteSwapInplace(src) {
const b = src.BYTES_PER_ELEMENT;
if (b === 1)
return; // no swapping needed
if (IS_NODE) {
// Use builtin methods for swapping if in Node environment
const bytes = Buffer.from(src.buffer, src.byteOffset, src.length * b);
if (b === 2)
bytes.swap16();
if (b === 4)
bytes.swap32();
if (b === 8)
bytes.swap64();
return;
}
// In browser, need to flip manually
// Adapted from https://github.com/zbjornson/node-bswap/blob/master/bswap.js
const flipper = new Uint8Array(src.buffer, src.byteOffset, src.length * b);
const numFlips = b / 2;
const endByteIndex = b - 1;
let t;
for (let i = 0; i < flipper.length; i += b) {
for (let j = 0; j < numFlips; j++) {
t = flipper[i + j];
flipper[i + j] = flipper[i + endByteIndex - j];
flipper[i + endByteIndex - j] = t;
}
}
}
/**
* Creates a copy of a TypedArray and swaps bytes.
* Used to flip endian-ness when getting/setting chunks from/to zarr store.
* @param src TypedArray
*/
function byteSwap(src) {
const copy = src.slice();
byteSwapInplace(copy);
return copy;
}
function convertColMajorToRowMajor2D(src, out, shape) {
let idx = 0;
const shape0 = shape[0];
const shape1 = shape[1];
const stride0 = shape1;
for (let i1 = 0; i1 < shape1; i1++) {
for (let i0 = 0; i0 < shape0; i0++) {
out[i0 * stride0 + i1] = src[idx++];
}
}
}
function convertColMajorToRowMajor3D(src, out, shape) {
let idx = 0;
const shape0 = shape[0];
const shape1 = shape[1];
const shape2 = shape[2];
const stride0 = shape2 * shape1;
const stride1 = shape2;
for (let i2 = 0; i2 < shape2; i2++) {
for (let i1 = 0; i1 < shape1; i1++) {
for (let i0 = 0; i0 < shape0; i0++) {
out[i0 * stride0 + i1 * stride1 + i2] = src[idx++];
}
}
}
}
function convertColMajorToRowMajor4D(src, out, shape) {
let idx = 0;
const shape0 = shape[0];
const shape1 = shape[1];
const shape2 = shape[2];
const shape3 = shape[3];
const stride0 = shape3 * shape2 * shape1;
const stride1 = shape3 * shape2;
const stride2 = shape3;
for (let i3 = 0; i3 < shape3; i3++) {
for (let i2 = 0; i2 < shape2; i2++) {
for (let i1 = 0; i1 < shape1; i1++) {
for (let i0 = 0; i0 < shape0; i0++) {
out[i0 * stride0 + i1 * stride1 + i2 * stride2 + i3] = src[idx++];
}
}
}
}
}
function convertColMajorToRowMajorGeneric(src, out, shape) {
const nDims = shape.length;
const size = shape.reduce((r, a) => r * a);
const rowMajorStrides = shape.map((_, i) => i + 1 === nDims ? 1 : shape.slice(i + 1).reduce((r, a) => r * a, 1));
const index = Array(nDims).fill(0);
for (let colMajorIdx = 0; colMajorIdx < size; colMajorIdx++) {
let rowMajorIdx = 0;
for (let dim = 0; dim < nDims; dim++) {
rowMajorIdx += index[dim] * rowMajorStrides[dim];
}
out[rowMajorIdx] = src[colMajorIdx];
index[0] += 1;
// Handle carry-over
for (let dim = 0; dim < nDims; dim++) {
if (index[dim] === shape[dim]) {
if (dim + 1 === nDims) {
return;
}
index[dim] = 0;
index[dim + 1] += 1;
}
}
}
}
const colMajorToRowMajorConverters = {
[0]: noop,
[1]: noop,
[2]: convertColMajorToRowMajor2D,
[3]: convertColMajorToRowMajor3D,
[4]: convertColMajorToRowMajor4D,
};
/**
* Rewrites a copy of a TypedArray while converting it from column-major (F-order) to row-major (C-order).
* @param src TypedArray
* @param out TypedArray
* @param shape number[]
*/
function convertColMajorToRowMajor(src, out, shape) {
return (colMajorToRowMajorConverters[shape.length] || convertColMajorToRowMajorGeneric)(src, out, shape);
}
function isArrayBufferLike(obj) {
if (obj === null) {
return false;
}
if (obj instanceof ArrayBuffer) {
return true;
}
if (typeof SharedArrayBuffer === "function" && obj instanceof SharedArrayBuffer) {
return true;
}
if (IS_NODE) { // Necessary for Node.js for some reason..
return obj.toString().startsWith("[object ArrayBuffer]")
|| obj.toString().startsWith("[object SharedArrayBuffer]");
}
return false;
}
const ARRAY_META_KEY = ".zarray";
const GROUP_META_KEY = ".zgroup";
const ATTRS_META_KEY = ".zattrs";
/**
* Return true if the store contains an array at the given logical path.
*/
async function containsArray(store, path = null) {
path = normalizeStoragePath(path);
const prefix = pathToPrefix(path);
const key = prefix + ARRAY_META_KEY;
return store.containsItem(key);
}
/**
* Return true if the store contains a group at the given logical path.
*/
async function containsGroup(store, path = null) {
path = normalizeStoragePath(path);
const prefix = pathToPrefix(path);
const key = prefix + GROUP_META_KEY;
return store.containsItem(key);
}
function pathToPrefix(path) {
// assume path already normalized
if (path.length > 0) {
return path + '/';
}
return '';
}
async function requireParentGroup(store, path, chunkStore, overwrite) {
// Assume path is normalized
if (path.length === 0) {
return;
}
const segments = path.split("/");
let p = "";
for (const s of segments.slice(0, segments.length - 1)) {
p += s;
if (await containsArray(store, p)) {
await initGroupMetadata(store, p, overwrite);
}
else if (!await containsGroup(store, p)) {
await initGroupMetadata(store, p);
}
p += "/";
}
}
async function initGroupMetadata(store, path = null, overwrite = false) {
path = normalizeStoragePath(path);
// Guard conditions
if (overwrite) {
throw Error("Group overwriting not implemented yet :(");
}
else if (await containsArray(store, path)) {
throw new ContainsArrayError(path);
}
else if (await containsGroup(store, path)) {
throw new ContainsGroupError(path);
}
const metadata = { zarr_format: 2 };
const key = pathToPrefix(path) + GROUP_META_KEY;
await store.setItem(key, JSON.stringify(metadata));
}
/**
* Initialize a group store. Note that this is a low-level function and there should be no
* need to call this directly from user code.
*/
async function initGroup(store, path = null, chunkStore = null, overwrite = false) {
path = normalizeStoragePath(path);
await requireParentGroup(store, path, chunkStore, overwrite);
await initGroupMetadata(store, path, overwrite);
}
async function initArrayMetadata(store, shape, chunks, dtype, path, compressor, fillValue, order, overwrite, chunkStore, filters, dimensionSeparator) {
// Guard conditions
if (overwrite) {
throw Error("Array overwriting not implemented yet :(");
}
else if (await containsArray(store, path)) {
throw new ContainsArrayError(path);
}
else if (await containsGroup(store, path)) {
throw new ContainsGroupError(path);
}
// Normalize metadata, does type checking too.
dtype = normalizeDtype(dtype);
shape = normalizeShape(shape);
chunks = normalizeChunks(chunks, shape);
order = normalizeOrder(order);
fillValue = normalizeFillValue(fillValue);
if (filters !== null && filters.length > 0) {
throw Error("Filters are not supported yet");
}
let serializedFillValue = fillValue;
if (typeof fillValue === "number") {
if (Number.isNaN(fillValue))
serializedFillValue = "NaN";
if (Number.POSITIVE_INFINITY === fillValue)
serializedFillValue = "Infinity";
if (Number.NEGATIVE_INFINITY === fillValue)
serializedFillValue = "-Infinity";
}
filters = null;
const metadata = {
zarr_format: 2,
shape: shape,
chunks: chunks,
dtype: dtype,
fill_value: serializedFillValue,
order: order,
compressor: compressor,
filters: filters,
};
if (dimensionSeparator) {
metadata.dimension_separator = dimensionSeparator;
}
const metaKey = pathToPrefix(path) + ARRAY_META_KEY;
await store.setItem(metaKey, JSON.stringify(metadata));
}
/**
*
* Initialize an array store with the given configuration. Note that this is a low-level
* function and there should be no need to call this directly from user code
*/
async function initArray(store, shape, chunks, dtype, path = null, compressor = null, fillValue = null, order = "C", overwrite = false, chunkStore = null, filters = null, dimensionSeparator) {
path = normalizeStoragePath(path);
await requireParentGroup(store, path, chunkStore, overwrite);
await initArrayMetadata(store, shape, chunks, dtype, path, compressor, fillValue, order, overwrite, chunkStore, filters, dimensionSeparator);
}
function parseMetadata(s) {
// Here we allow that a store may return an already-parsed metadata object,
// or a string of JSON that we will parse here. We allow for an already-parsed
// object to accommodate a consolidated metadata store, where all the metadata for
// all groups and arrays will already have been parsed from JSON.
if (typeof s !== 'string') {
// tslint:disable-next-line: strict-type-predicates
if (IS_NODE && Buffer.isBuffer(s)) {
return JSON.parse(s.toString());
}
else if (isArrayBufferLike(s)) {
const utf8Decoder = new TextDecoder();
const bytes = new Uint8Array(s);
return JSON.parse(utf8Decoder.decode(bytes));
}
else {
return s;
}
}
return JSON.parse(s);
}
/**
* Class providing access to user attributes on an array or group. Should not be
* instantiated directly, will be available via the `.attrs` property of an array or
* group.
*/
class Attributes {
constructor(store, key, readOnly, cache = true) {
this.store = store;
this.key = key;
this.readOnly = readOnly;
this.cache = cache;
this.cachedValue = null;
}
/**
* Retrieve all attributes as a JSON object.
*/
async asObject() {
if (this.cache && this.cachedValue !== null) {
return this.cachedValue;
}
const o = await this.getNoSync();
if (this.cache) {
this.cachedValue = o;
}
return o;
}
async getNoSync() {
try {
const data = await this.store.getItem(this.key);
// TODO fix typing?
return parseMetadata(data);
}
catch (error) {
return {};
}
}
async setNoSync(key, value) {
const d = await this.getNoSync();
d[key] = value;
await this.putNoSync(d);
return true;
}
async putNoSync(m) {
await this.store.setItem(this.key, JSON.stringify(m));
if (this.cache) {
this.cachedValue = m;
}
}
async delNoSync(key) {
const d = await this.getNoSync();
delete d[key];
await this.putNoSync(d);
return true;
}
/**
* Overwrite all attributes with the provided object in a single operation
*/
async put(d) {
if (this.readOnly) {
throw new PermissionError("attributes are read-only");
}
return this.putNoSync(d);
}
async setItem(key, value) {
if (this.readOnly) {
throw new PermissionError("attributes are read-only");
}
return this.setNoSync(key, value);
}
async getItem(key) {
return (await this.asObject())[key];
}
async deleteItem(key) {
if (this.readOnly) {
throw new PermissionError("attributes are read-only");
}
return this.delNoSync(key);
}
async containsItem(key) {
return (await this.asObject())[key] !== undefined;
}
proxy() {
return createProxy(this);
}
}
// eslint-disable-next-line @typescript-eslint/naming-convention
const Float16Array = globalThis.Float16Array;
const DTYPE_TYPEDARRAY_MAPPING = {
'|b': Int8Array,
'|b1': Uint8Array,
'|B': Uint8Array,
'|u1': Uint8Array,
'|i1': Int8Array,
'<b': Int8Array,
'<B': Uint8Array,
'<u1': Uint8Array,
'<i1': Int8Array,
'<u2': Uint16Array,
'<i2': Int16Array,
'<u4': Uint32Array,
'<i4': Int32Array,
'<f4': Float32Array,
'<f2': Float16Array,
'<f8': Float64Array,
'>b': Int8Array,
'>B': Uint8Array,
'>u1': Uint8Array,
'>i1': Int8Array,
'>u2': Uint16Array,
'>i2': Int16Array,
'>u4': Uint32Array,
'>i4': Int32Array,
'>f4': Float32Array,
'>f2': Float16Array,
'>f8': Float64Array
};
function getTypedArrayCtr(dtype) {
const ctr = DTYPE_TYPEDARRAY_MAPPING[dtype];
if (!ctr) {
if (dtype.slice(1) === 'f2') {
throw Error(`'${dtype}' is not supported natively in zarr.js. ` +
`In order to access this dataset you must make Float16Array available as a global. ` +
`See https://github.com/gzuidhof/zarr.js/issues/127`);
}
throw Error(`Dtype not recognized or not supported in zarr.js, got ${dtype}.`);
}
return ctr;
}
/*
* Called by NestedArray and RawArray constructors only.
* We byte-swap the buffer of a store after decoding
* since TypedArray views are little endian only.
*
* This means NestedArrays and RawArrays will always be little endian,
* unless a numpy-like library comes around and can handle endianess
* for buffer views.
*/
function getTypedArrayDtypeString(t) {
// Favour the types below instead of small and big B
if (t instanceof Uint8Array)
return '|u1';
if (t instanceof Int8Array)
return '|i1';
if (t instanceof Uint16Array)
return '<u2';
if (t instanceof Int16Array)
return '<i2';
if (t instanceof Uint32Array)
return '<u4';
if (t instanceof Int32Array)
return '<i4';
if (t instanceof Float32Array)
return '<f4';
if (t instanceof Float64Array)
return '<f8';
throw new ValueError('Mapping for TypedArray to Dtypestring not known');
}
/**
* Digs down into the dimensions of given array to find the TypedArray and returns its constructor.
* Better to use sparingly.
*/
function getNestedArrayConstructor(arr) {
// TODO fix typing
// tslint:disable-next-line: strict-type-predicates
if (arr.byteLength !== undefined) {
return (arr).constructor;
}
return getNestedArrayConstructor(arr[0]);
}
/**
* Returns both the slice result and new output shape
* @param arr NestedArray to slice
* @param shape The shape of the NestedArray
* @param selection
*/
function sliceNestedArray(arr, shape, selection) {
// This translates "...", ":", null into a list of slices or integer selections
const normalizedSelection = normalizeArraySelection(selection, shape);
const [sliceIndices, outShape] = selectionToSliceIndices(normalizedSelection, shape);
const outArray = _sliceNestedArray(arr, shape, sliceIndices);
return [outArray, outShape];
}
function _sliceNestedArray(arr, shape, selection) {
const currentSlice = selection[0];
// Is this necessary?
// // This is possible when a slice list is passed shorter than the amount of dimensions
// // tslint:disable-next-line: strict-type-predicates
// if (currentSlice === undefined) {
// return arr.slice();
// }
// When a number is passed that dimension is squeezed
if (typeof currentSlice === "number") {
// Assume already normalized integer selection here.
if (shape.length === 1) {
return arr[currentSlice];
}
else {
return _sliceNestedArray(arr[currentSlice], shape.slice(1), selection.slice(1));
}
}
const [from, to, step, outputSize] = currentSlice;
if (outputSize === 0) {
return new (getNestedArrayConstructor(arr))(0);
}
if (shape.length === 1) {
if (step === 1) {
return arr.slice(from, to);
}
const newArrData = new arr.constructor(outputSize);
for (let i = 0; i < outputSize; i++) {
newArrData[i] = arr[from + i * step];
}
return newArrData;
}
let newArr = new Array(outputSize);
for (let i = 0; i < outputSize; i++) {
newArr[i] = _sliceNestedArray(arr[from + i * step], shape.slice(1), selection.slice(1));
}
// This is necessary to ensure that the return value is a NestedArray if the last dimension is squeezed
// e.g. shape [2,1] with slice [:, 0] would otherwise result in a list of numbers instead of a valid NestedArray
if (outputSize > 0 && typeof newArr[0] === "number") {
const typedArrayConstructor = arr[0].constructor;
newArr = typedArrayConstructor.from(newArr);
}
return newArr;
}
function setNestedArrayToScalar(dstArr, value, destShape, selection) {
// This translates "...", ":", null, etc into a list of slices.
const normalizedSelection = normalizeArraySelection(selection, destShape, true);
// Above we force the results to be SliceIndicesIndices only, without integer selections making this cast is safe.
const [sliceIndices, _outShape] = selectionToSliceIndices(normalizedSelection, destShape);
_setNestedArrayToScalar(dstArr, value, destShape, sliceIndices);
}
function setNestedArray(dstArr, sourceArr, destShape, sourceShape, selection) {
// This translates "...", ":", null, etc into a list of slices.
const normalizedSelection = normalizeArraySelection(selection, destShape, false);
const [sliceIndices, outShape] = selectionToSliceIndices(normalizedSelection, destShape);
// TODO: replace with non stringify equality check
if (JSON.stringify(outShape) !== JSON.stringify(sourceShape)) {
throw new ValueError(`Shape mismatch in target and source NestedArray: ${outShape} and ${sourceShape}`);
}
_setNestedArray(dstArr, sourceArr, destShape, sliceIndices);
}
function _setNestedArray(dstArr, sourceArr, shape, selection) {
const currentSlice = selection[0];
if (typeof sourceArr === "number") {
_setNestedArrayToScalar(dstArr, sourceArr, shape, selection.map(x => typeof x === "number" ? [x, x + 1, 1, 1] : x));
return;
}
// This dimension is squeezed.
if (typeof currentSlice === "number") {
_setNestedArray(dstArr[currentSlice], sourceArr, shape.slice(1), selection.slice(1));
return;
}
const [from, _to, step, outputSize] = currentSlice;
if (shape.length === 1) {
if (step === 1) {
dstArr.set(sourceArr, from);
}
else {
for (let i = 0; i < outputSize; i++) {
dstArr[from + i * step] = (sourceArr)[i];
}
}
return;
}
for (let i = 0; i < outputSize; i++) {
_setNestedArray(dstArr[from + i * step], sourceArr[i], shape.slice(1), selection.slice(1));
}
}
function _setNestedArrayToScalar(dstArr, value, shape, selection) {
const currentSlice = selection[0];
const [from, to, step, outputSize] = currentSlice;
if (shape.length === 1) {
if (step === 1) {
dstArr.fill(value, from, to);
}
else {
for (let i = 0; i < outputSize; i++) {
dstArr[from + i * step] = value;
}
}
return;
}
for (let i = 0; i < outputSize; i++) {
_setNestedArrayToScalar(dstArr[from + i * step], value, shape.slice(1), selection.slice(1));
}
}
function flattenNestedArray(arr, shape, constr) {
if (constr === undefined) {
constr = getNestedArrayConstructor(arr);
}
const size = shape.reduce((x, y) => x * y, 1);
const outArr = new constr(size);
_flattenNestedArray(arr, shape, outArr, 0);
return outArr;
}
function _flattenNestedArray(arr, shape, outArr, offset) {
if (shape.length === 1) {
// This is only ever reached if called with rank 1 shape, never reached through recursion.
// We just slice set the array directly from one level above to save some function calls.
outArr.set(arr, offset);
return;
}
if (shape.length === 2) {
for (let i = 0; i < shape[0]; i++) {
outArr.set(arr[i], offset + shape[1] * i);
}
return arr;
}
const nextShape = shape.slice(1);
// Small optimization possible here: this can be precomputed for different levels of depth and passed on.
const mult = nextShape.reduce((x, y) => x * y, 1);
for (let i = 0; i < shape[0]; i++) {
_flattenNestedArray(arr[i], nextShape, outArr, offset + mult * i);
}
return arr;
}
class NestedArray {
constructor(data, shape, dtype) {
const dataIsTypedArray = data !== null && !!data.BYTES_PER_ELEMENT;
if (shape === undefined) {
if (!dataIsTypedArray) {
throw new ValueError("Shape argument is required unless you pass in a TypedArray");
}
shape = [data.length];
}
if (dtype === undefined) {
if (!dataIsTypedArray) {
throw new ValueError("Dtype argument is required unless you pass in a TypedArray");
}
dtype = getTypedArrayDtypeString(data);
}
shape = normalizeShape(shape);
this.shape = shape;
this.dtype = dtype;
if (dataIsTypedArray && shape.length !== 1) {
data = data.buffer;
}
// Zero dimension array.. they are a bit weirdly represented now, they will only ever occur internally
if (this.shape.length === 0) {
this.data = new (getTypedArrayCtr(dtype))(1);
}
else if (
// tslint:disable-next-line: strict-type-predicates
(IS_NODE && Buffer.isBuffer(data))
|| isArrayBufferLike(data)
|| data === null) {
// Create from ArrayBuffer or Buffer
const numShapeElements = shape.reduce((x, y) => x * y, 1);
if (data === null) {
data = new ArrayBuffer(numShapeElements * parseInt(dtype[dtype.length - 1], 10));
}
const numDataElements = data.byteLength / parseInt(dtype[dtype.length - 1], 10);
if (numShapeElements !== numDataElements) {
throw new Error(`Buffer has ${numDataElements} of dtype ${dtype}, shape is too large or small ${shape} (flat=${numShapeElements})`);
}
const typeConstructor = getTypedArrayCtr(dtype);
this.data = createNestedArray(data, typeConstructor, shape);
}
else {
this.data = data;
}
}
get(selection) {
const [sliceResult, outShape] = sliceNestedArray(this.data, this.shape, selection);
if (outShape.length === 0) {
return sliceResult;
}
else {
return new NestedArray(sliceResult, outShape, this.dtype);
}
}
set(selection = null, value) {
if (selection === null) {
selection = [slice(null)];
}
if (typeof value === "number") {
if (this.shape.length === 0) {
// Zero dimension array..
this.data[0] = value;
}
else {
setNestedArrayToScalar(this.data, value, this.shape, selection);
}
}
else {
setNestedArray(this.data, value.data, this.shape, value.shape, selection);
}
}
flatten() {
if (this.shape.length === 1) {
return this.data;
}
return flattenNestedArray(this.data, this.shape, getTypedArrayCtr(this.dtype));
}
/**
* Currently only supports a single integer as the size, TODO: support start, stop, step.
*/
static arange(size, dtype = "<i4") {
const constr = getTypedArrayCtr(dtype);
const data = rangeTypedArray([size], constr);
return new NestedArray(data, [size], dtype);
}
}
/**
* Creates a TypedArray with values 0 through N where N is the product of the shape.
*/
function rangeTypedArray(shape, tContructor) {
const size = shape.reduce((x, y) => x * y, 1);
const data = new tContructor(size);
data.set([...Array(size).keys()]); // Sets range 0,1,2,3,4,5
return data;
}
/**
* Creates multi-dimensional (rank > 1) array given input data and shape recursively.
* What it does is create a Array<Array<...<Array<Uint8Ar