apache-arrow
Version:
Apache Arrow columnar in-memory format
356 lines (354 loc) • 13.6 kB
JavaScript
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
var _a;
import { Type } from './enum.mjs';
import { Data, makeData } from './data.mjs';
import { vectorFromArray } from './factories.mjs';
import { makeVector, Vector } from './vector.mjs';
import { Field, Schema } from './schema.mjs';
import { Null, Struct } from './type.mjs';
import { compareSchemas } from './visitor/typecomparator.mjs';
import { distributeVectorsIntoRecordBatches } from './util/recordbatch.mjs';
import { isChunkedValid, computeChunkOffsets, computeChunkNullCounts, wrapChunkedCall1, wrapChunkedCall2, wrapChunkedIndexOf, sliceChunks, } from './util/chunk.mjs';
import { instance as getVisitor } from './visitor/get.mjs';
import { instance as setVisitor } from './visitor/set.mjs';
import { instance as indexOfVisitor } from './visitor/indexof.mjs';
import { instance as iteratorVisitor } from './visitor/iterator.mjs';
import { clampRange, wrapIndex } from './util/vector.mjs';
import { RecordBatch } from './recordbatch.mjs';
/**
* Tables are collections of {@link Vector}s and have a {@link Schema}. Use the convenience methods {@link makeTable}
* or {@link tableFromArrays} to create a table in JavaScript. To create a table from the IPC format, use
* {@link tableFromIPC}.
*/
export class Table {
constructor(...args) {
var _b, _c;
if (args.length === 0) {
this.batches = [];
this.schema = new Schema([]);
this._offsets = [0];
return this;
}
let schema;
let offsets;
if (args[0] instanceof Schema) {
schema = args.shift();
}
if (args.at(-1) instanceof Uint32Array) {
offsets = args.pop();
}
const unwrap = (x) => {
if (x) {
if (x instanceof RecordBatch) {
return [x];
}
else if (x instanceof Table) {
return x.batches;
}
else if (x instanceof Data) {
if (x.type instanceof Struct) {
return [new RecordBatch(new Schema(x.type.children), x)];
}
}
else if (Array.isArray(x)) {
return x.flatMap(v => unwrap(v));
}
else if (typeof x[Symbol.iterator] === 'function') {
return [...x].flatMap(v => unwrap(v));
}
else if (typeof x === 'object') {
const keys = Object.keys(x);
const vecs = keys.map((k) => new Vector([x[k]]));
const batchSchema = schema !== null && schema !== void 0 ? schema : new Schema(keys.map((k, i) => new Field(String(k), vecs[i].type, vecs[i].nullable)));
const [, batches] = distributeVectorsIntoRecordBatches(batchSchema, vecs);
return batches.length === 0 ? [new RecordBatch(x)] : batches;
}
}
return [];
};
const batches = args.flatMap(v => unwrap(v));
schema = (_c = schema !== null && schema !== void 0 ? schema : (_b = batches[0]) === null || _b === void 0 ? void 0 : _b.schema) !== null && _c !== void 0 ? _c : new Schema([]);
if (!(schema instanceof Schema)) {
throw new TypeError('Table constructor expects a [Schema, RecordBatch[]] pair.');
}
for (const batch of batches) {
if (!(batch instanceof RecordBatch)) {
throw new TypeError('Table constructor expects a [Schema, RecordBatch[]] pair.');
}
if (!compareSchemas(schema, batch.schema)) {
throw new TypeError('Table and inner RecordBatch schemas must be equivalent.');
}
}
this.schema = schema;
this.batches = batches;
this._offsets = offsets !== null && offsets !== void 0 ? offsets : computeChunkOffsets(this.data);
}
/**
* The contiguous {@link RecordBatch `RecordBatch`} chunks of the Table rows.
*/
get data() { return this.batches.map(({ data }) => data); }
/**
* The number of columns in this Table.
*/
get numCols() { return this.schema.fields.length; }
/**
* The number of rows in this Table.
*/
get numRows() {
return this.data.reduce((numRows, data) => numRows + data.length, 0);
}
/**
* The number of null rows in this Table.
*/
get nullCount() {
if (this._nullCount === -1) {
this._nullCount = computeChunkNullCounts(this.data);
}
return this._nullCount;
}
/**
* Check whether an element is null.
*
* @param index The index at which to read the validity bitmap.
*/
// @ts-ignore
isValid(index) { return false; }
/**
* Get an element value by position.
*
* @param index The index of the element to read.
*/
// @ts-ignore
get(index) { return null; }
/**
* Get an element value by position.
* @param index The index of the element to read. A negative index will count back from the last element.
*/
// @ts-ignore
at(index) {
return this.get(wrapIndex(index, this.numRows));
}
/**
* Set an element value by position.
*
* @param index The index of the element to write.
* @param value The value to set.
*/
// @ts-ignore
set(index, value) { return; }
/**
* Retrieve the index of the first occurrence of a value in an Vector.
*
* @param element The value to locate in the Vector.
* @param offset The index at which to begin the search. If offset is omitted, the search starts at index 0.
*/
// @ts-ignore
indexOf(element, offset) { return -1; }
/**
* Iterator for rows in this Table.
*/
[Symbol.iterator]() {
if (this.batches.length > 0) {
return iteratorVisitor.visit(new Vector(this.data));
}
return (new Array(0))[Symbol.iterator]();
}
/**
* Return a JavaScript Array of the Table rows.
*
* @returns An Array of Table rows.
*/
toArray() {
return [...this];
}
/**
* Returns a string representation of the Table rows.
*
* @returns A string representation of the Table rows.
*/
toString() {
return `[\n ${this.toArray().join(',\n ')}\n]`;
}
/**
* Combines two or more Tables of the same schema.
*
* @param others Additional Tables to add to the end of this Tables.
*/
concat(...others) {
const schema = this.schema;
const data = this.data.concat(others.flatMap(({ data }) => data));
return new Table(schema, data.map((data) => new RecordBatch(schema, data)));
}
/**
* Return a zero-copy sub-section of this Table.
*
* @param begin The beginning of the specified portion of the Table.
* @param end The end of the specified portion of the Table. This is exclusive of the element at the index 'end'.
*/
slice(begin, end) {
const schema = this.schema;
[begin, end] = clampRange({ length: this.numRows }, begin, end);
const data = sliceChunks(this.data, this._offsets, begin, end);
return new Table(schema, data.map((chunk) => new RecordBatch(schema, chunk)));
}
/**
* Returns a child Vector by name, or null if this Vector has no child with the given name.
*
* @param name The name of the child to retrieve.
*/
getChild(name) {
return this.getChildAt(this.schema.fields.findIndex((f) => f.name === name));
}
/**
* Returns a child Vector by index, or null if this Vector has no child at the supplied index.
*
* @param index The index of the child to retrieve.
*/
getChildAt(index) {
if (index > -1 && index < this.schema.fields.length) {
const data = this.data.map((data) => data.children[index]);
if (data.length === 0) {
const { type } = this.schema.fields[index];
const empty = makeData({ type, length: 0, nullCount: 0 });
data.push(empty._changeLengthAndBackfillNullBitmap(this.numRows));
}
return new Vector(data);
}
return null;
}
/**
* Sets a child Vector by name.
*
* @param name The name of the child to overwrite.
* @returns A new Table with the supplied child for the specified name.
*/
setChild(name, child) {
var _b;
return this.setChildAt((_b = this.schema.fields) === null || _b === void 0 ? void 0 : _b.findIndex((f) => f.name === name), child);
}
setChildAt(index, child) {
let schema = this.schema;
let batches = [...this.batches];
if (index > -1 && index < this.numCols) {
if (!child) {
child = new Vector([makeData({ type: new Null, length: this.numRows })]);
}
const fields = schema.fields.slice();
const field = fields[index].clone({ type: child.type });
const children = this.schema.fields.map((_, i) => this.getChildAt(i));
[fields[index], children[index]] = [field, child];
[schema, batches] = distributeVectorsIntoRecordBatches(schema, children);
}
return new Table(schema, batches);
}
/**
* Construct a new Table containing only specified columns.
*
* @param columnNames Names of columns to keep.
* @returns A new Table of columns matching the specified names.
*/
select(columnNames) {
const nameToIndex = this.schema.fields.reduce((m, f, i) => m.set(f.name, i), new Map());
return this.selectAt(columnNames.map((columnName) => nameToIndex.get(columnName)).filter((x) => x > -1));
}
/**
* Construct a new Table containing only columns at the specified indices.
*
* @param columnIndices Indices of columns to keep.
* @returns A new Table of columns at the specified indices.
*/
selectAt(columnIndices) {
const schema = this.schema.selectAt(columnIndices);
const data = this.batches.map((batch) => batch.selectAt(columnIndices));
return new Table(schema, data);
}
assign(other) {
const fields = this.schema.fields;
const [indices, oldToNew] = other.schema.fields.reduce((memo, f2, newIdx) => {
const [indices, oldToNew] = memo;
const i = fields.findIndex((f) => f.name === f2.name);
~i ? (oldToNew[i] = newIdx) : indices.push(newIdx);
return memo;
}, [[], []]);
const schema = this.schema.assign(other.schema);
const columns = [
...fields.map((_, i) => [i, oldToNew[i]]).map(([i, j]) => (j === undefined ? this.getChildAt(i) : other.getChildAt(j))),
...indices.map((i) => other.getChildAt(i))
].filter(Boolean);
return new Table(...distributeVectorsIntoRecordBatches(schema, columns));
}
}
_a = Symbol.toStringTag;
// Initialize this static property via an IIFE so bundlers don't tree-shake
// out this logic, but also so we're still compliant with `"sideEffects": false`
Table[_a] = ((proto) => {
proto.schema = null;
proto.batches = [];
proto._offsets = new Uint32Array([0]);
proto._nullCount = -1;
proto[Symbol.isConcatSpreadable] = true;
proto['isValid'] = wrapChunkedCall1(isChunkedValid);
proto['get'] = wrapChunkedCall1(getVisitor.getVisitFn(Type.Struct));
proto['set'] = wrapChunkedCall2(setVisitor.getVisitFn(Type.Struct));
proto['indexOf'] = wrapChunkedIndexOf(indexOfVisitor.getVisitFn(Type.Struct));
return 'Table';
})(Table.prototype);
/**
* Creates a new Table from an object of typed arrays.
*
* @example
* ```ts
* const table = makeTable({
* a: new Int8Array([1, 2, 3]),
* })
* ```
*
* @param input Input an object of typed arrays.
* @returns A new Table.
*/
export function makeTable(input) {
const vecs = {};
const inputs = Object.entries(input);
for (const [key, col] of inputs) {
vecs[key] = makeVector(col);
}
return new Table(vecs);
}
/**
* Creates a new Table from an object of typed arrays or JavaScript arrays.
*
* @example
* ```ts
* const table = tableFromArrays({
* a: [1, 2, 3],
* b: new Int8Array([1, 2, 3]),
* })
* ```
*
* @param input Input an object of typed arrays or JavaScript arrays.
* @returns A new Table.
*/
export function tableFromArrays(input) {
const vecs = {};
const inputs = Object.entries(input);
for (const [key, col] of inputs) {
vecs[key] = vectorFromArray(col);
}
return new Table(vecs);
}
//# sourceMappingURL=table.mjs.map