react-native-avsc
Version:
Javascript umd module for avsc
1,873 lines (1,601 loc) • 514 kB
JavaScript
(function webpackUniversalModuleDefinition(root, factory) {
if(typeof exports === 'object' && typeof module === 'object')
module.exports = factory(require("buffer"), require("stream-browserify"));
else if(typeof define === 'function' && define.amd)
define(["buffer", "stream-browserify"], factory);
else if(typeof exports === 'object')
exports["avsc"] = factory(require("buffer"), require("stream-browserify"));
else
root["avsc"] = factory(root["buffer"], root["stream-browserify"]);
})(this, function(__WEBPACK_EXTERNAL_MODULE_8__, __WEBPACK_EXTERNAL_MODULE_12__) {
return /******/ (function(modules) { // webpackBootstrap
/******/ // The module cache
/******/ var installedModules = {};
/******/ // The require function
/******/ function __webpack_require__(moduleId) {
/******/ // Check if module is in cache
/******/ if(installedModules[moduleId])
/******/ return installedModules[moduleId].exports;
/******/ // Create a new module (and put it into the cache)
/******/ var module = installedModules[moduleId] = {
/******/ exports: {},
/******/ id: moduleId,
/******/ loaded: false
/******/ };
/******/ // Execute the module function
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
/******/ // Flag the module as loaded
/******/ module.loaded = true;
/******/ // Return the exports of the module
/******/ return module.exports;
/******/ }
/******/ // expose the modules object (__webpack_modules__)
/******/ __webpack_require__.m = modules;
/******/ // expose the module cache
/******/ __webpack_require__.c = installedModules;
/******/ // __webpack_public_path__
/******/ __webpack_require__.p = "";
/******/ // Load entry module and return exports
/******/ return __webpack_require__(0);
/******/ })
/************************************************************************/
/******/ ([
/* 0 */
/***/ function(module, exports, __webpack_require__) {
// Get original avsc
var avscWrapper = __webpack_require__(1);
// Delete decodeFile because we don't want to read files in browser
delete avscWrapper.decodeFile;
// Delete getFileHeader because we don't want to read files in browser
delete avscWrapper.getFileHeader;
// Export
module.exports = avscWrapper;
/***/ },
/* 1 */
/***/ function(module, exports, __webpack_require__) {
/* jshint node: true */
'use strict';
/**
* Node.js entry point (see `etc/browser/` for browserify's entry points).
*
* It also adds Node.js specific functionality (for example a few convenience
* functions to read Avro files from the local filesystem).
*
*/
var containers = __webpack_require__(2),
files = __webpack_require__(3),
protocols = __webpack_require__(38),
schemas = __webpack_require__(40),
types = __webpack_require__(5),
utils = __webpack_require__(6),
values = __webpack_require__(42),
fs = __webpack_require__(4);
/**
* Parse a schema and return the corresponding type or protocol.
*
*/
function parse(schema, opts) {
var attrs = files.load(schema);
return attrs.protocol ?
protocols.createProtocol(attrs, opts) :
types.createType(attrs, opts);
}
/**
* Extract a container file's header synchronously.
*
*/
function extractFileHeader(path, opts) {
opts = opts || {};
var decode = opts.decode === undefined ? true : !!opts.decode;
var size = Math.max(opts.size || 4096, 4);
var fd = fs.openSync(path, 'r');
var buf = new Buffer(size);
var pos = 0;
var tap = new utils.Tap(buf);
var header = null;
while (pos < 4) {
// Make sure we have enough to check the magic bytes.
pos += fs.readSync(fd, buf, pos, size - pos);
}
if (containers.MAGIC_BYTES.equals(buf.slice(0, 4))) {
do {
header = containers.HEADER_TYPE._read(tap);
} while (!isValid());
if (decode !== false) {
var meta = header.meta;
meta['avro.schema'] = JSON.parse(meta['avro.schema'].toString());
if (meta['avro.codec'] !== undefined) {
meta['avro.codec'] = meta['avro.codec'].toString();
}
}
}
fs.closeSync(fd);
return header;
function isValid() {
if (tap.isValid()) {
return true;
}
var len = 2 * tap.buf.length;
var buf = new Buffer(len);
len = fs.readSync(fd, buf, 0, len);
tap.buf = Buffer.concat([tap.buf, buf]);
tap.pos = 0;
return false;
}
}
/**
* Readable stream of records from a local Avro file.
*
*/
function createFileDecoder(path, opts) {
return fs.createReadStream(path)
.pipe(new containers.streams.BlockDecoder(opts));
}
/**
* Writable stream of records to a local Avro file.
*
*/
function createFileEncoder(path, schema, opts) {
var encoder = new containers.streams.BlockEncoder(schema, opts);
encoder.pipe(fs.createWriteStream(path, {defaultEncoding: 'binary'}));
return encoder;
}
module.exports = {
Protocol: protocols.Protocol,
Type: types.Type,
assemble: schemas.assemble,
combine: values.combine,
createFileDecoder: createFileDecoder,
createFileEncoder: createFileEncoder,
extractFileHeader: extractFileHeader,
infer: values.infer,
parse: parse,
streams: containers.streams,
types: types.builtins
};
/***/ },
/* 2 */
/***/ function(module, exports, __webpack_require__) {
/* jshint node: true */
// TODO: Add streams which prefix each record with its length.
'use strict';
/**
* This module defines custom streams to write and read Avro files.
*
* In particular, the `Block{En,De}coder` streams are able to deal with Avro
* container files. None of the streams below depend on the filesystem however,
* this way they can also be used in the browser (for example to parse HTTP
* responses).
*
*/
var files = __webpack_require__(3),
types = __webpack_require__(5),
utils = __webpack_require__(6),
stream = __webpack_require__(12),
util = __webpack_require__(9),
zlib = __webpack_require__(13);
// Type of Avro header.
var HEADER_TYPE = types.createType({
namespace: 'org.apache.avro.file',
name: 'Header',
type: 'record',
fields : [
{name: 'magic', type: {type: 'fixed', name: 'Magic', size: 4}},
{name: 'meta', type: {type: 'map', values: 'bytes'}},
{name: 'sync', type: {type: 'fixed', name: 'Sync', size: 16}}
]
});
// Type of each block.
var BLOCK_TYPE = types.createType({
namespace: 'org.apache.avro.file',
name: 'Block',
type: 'record',
fields : [
{name: 'count', type: 'long'},
{name: 'data', type: 'bytes'},
{name: 'sync', type: {type: 'fixed', name: 'Sync', size: 16}}
]
});
// Used to toBuffer each block, without having to copy all its data.
var LONG_TYPE = types.createType('long');
// First 4 bytes of an Avro object container file.
var MAGIC_BYTES = new Buffer('Obj\x01');
// Convenience.
var f = util.format;
var Tap = utils.Tap;
/**
* Duplex stream for decoding fragments.
*
*/
function RawDecoder(schema, opts) {
opts = opts || {};
var noDecode = !!opts.noDecode;
stream.Duplex.call(this, {
readableObjectMode: !noDecode,
allowHalfOpen: false
});
this._type = types.createType(files.load(schema));
this._tap = new Tap(new Buffer(0));
this._writeCb = null;
this._needPush = false;
this._readValue = createReader(noDecode, this._type);
this._finished = false;
this.on('finish', function () {
this._finished = true;
this._read();
});
}
util.inherits(RawDecoder, stream.Duplex);
RawDecoder.prototype._write = function (chunk, encoding, cb) {
// Store the write callback and call it when we are done decoding all records
// in this chunk. If we call it right away, we risk loading the entire input
// in memory. We only need to store the latest callback since the stream API
// guarantees that `_write` won't be called again until we call the previous.
this._writeCb = cb;
var tap = this._tap;
tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]);
tap.pos = 0;
if (this._needPush) {
this._needPush = false;
this._read();
}
};
RawDecoder.prototype._read = function () {
this._needPush = false;
var tap = this._tap;
var pos = tap.pos;
var val = this._readValue(tap);
if (tap.isValid()) {
this.push(val);
} else if (!this._finished) {
tap.pos = pos;
this._needPush = true;
if (this._writeCb) {
// This should only ever be false on the first read, and only if it
// happens before the first write.
this._writeCb();
}
} else {
this.push(null);
}
};
/**
* Duplex stream for decoding object container files.
*
*/
function BlockDecoder(opts) {
opts = opts || {};
var noDecode = !!opts.noDecode;
stream.Duplex.call(this, {
allowHalfOpen: true, // For async decompressors.
readableObjectMode: !noDecode
});
this._type = null;
this._codecs = opts.codecs;
this._parseHook = opts.parseHook;
this._tap = new Tap(new Buffer(0));
this._blockTap = new Tap(new Buffer(0));
this._syncMarker = null;
this._readValue = null;
this._noDecode = noDecode;
this._queue = new utils.OrderedQueue();
this._decompress = null; // Decompression function.
this._index = 0; // Next block index.
this._needPush = false;
this._finished = false;
this.on('finish', function () {
this._finished = true;
if (this._needPush) {
this._read();
}
});
}
util.inherits(BlockDecoder, stream.Duplex);
BlockDecoder.getDefaultCodecs = function () {
return {
'null': function (buf, cb) { cb(null, buf); },
'deflate': zlib.inflateRaw
};
};
BlockDecoder.prototype._decodeHeader = function () {
var tap = this._tap;
if (tap.buf.length < MAGIC_BYTES.length) {
// Wait until more data arrives.
return false;
}
if (!MAGIC_BYTES.equals(tap.buf.slice(0, MAGIC_BYTES.length))) {
this.emit('error', new Error('invalid magic bytes'));
return false;
}
var header = HEADER_TYPE._read(tap);
if (!tap.isValid()) {
return false;
}
var codec = (header.meta['avro.codec'] || 'null').toString();
this._decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec];
if (!this._decompress) {
this.emit('error', new Error(f('unknown codec: %s', codec)));
return;
}
try {
var schema = JSON.parse(header.meta['avro.schema'].toString());
if (this._parseHook) {
schema = this._parseHook(schema);
}
this._type = types.createType(schema);
} catch (err) {
this.emit('error', err);
return;
}
this._readValue = createReader(this._noDecode, this._type);
this._syncMarker = header.sync;
this.emit('metadata', this._type, codec, header);
return true;
};
BlockDecoder.prototype._write = function (chunk, encoding, cb) {
var tap = this._tap;
tap.buf = Buffer.concat([tap.buf, chunk]);
tap.pos = 0;
if (!this._decodeHeader()) {
process.nextTick(cb);
return;
}
// We got the header, switch to block decoding mode. Also, call it directly
// in case we already have all the data (in which case `_write` wouldn't get
// called anymore).
this._write = this._writeChunk;
this._write(new Buffer(0), encoding, cb);
};
BlockDecoder.prototype._writeChunk = function (chunk, encoding, cb) {
var tap = this._tap;
tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]);
tap.pos = 0;
var nBlocks = 1;
var block;
while ((block = tryReadBlock(tap))) {
if (!this._syncMarker.equals(block.sync)) {
this.emit('error', new Error('invalid sync marker'));
return;
}
nBlocks++;
this._decompress(block.data, this._createBlockCallback(chunkCb));
}
chunkCb();
function chunkCb() {
if (!--nBlocks) {
cb();
}
}
};
BlockDecoder.prototype._createBlockCallback = function (cb) {
var self = this;
var index = this._index++;
return function (err, data) {
if (err) {
self.emit('error', err);
cb();
} else {
self._queue.push(new BlockData(index, data, cb));
if (self._needPush) {
self._read();
}
}
};
};
BlockDecoder.prototype._read = function () {
this._needPush = false;
var tap = this._blockTap;
if (tap.pos >= tap.buf.length) {
var data = this._queue.pop();
if (!data) {
if (this._finished) {
this.push(null);
} else {
this._needPush = true;
}
return; // Wait for more data.
}
data.cb();
tap.buf = data.buf;
tap.pos = 0;
}
this.push(this._readValue(tap)); // The read is guaranteed valid.
};
/**
* Duplex stream for encoding.
*
*/
function RawEncoder(schema, opts) {
opts = opts || {};
stream.Transform.call(this, {
writableObjectMode: true,
allowHalfOpen: false
});
this._type = types.createType(files.load(schema));
this._writeValue = function (tap, val) {
try {
this._type._write(tap, val);
} catch (err) {
this.emit('error', err);
}
};
this._tap = new Tap(new Buffer(opts.batchSize || 65536));
}
util.inherits(RawEncoder, stream.Transform);
RawEncoder.prototype._transform = function (val, encoding, cb) {
var tap = this._tap;
var buf = tap.buf;
var pos = tap.pos;
this._writeValue(tap, val);
if (!tap.isValid()) {
if (pos) {
// Emit any valid data.
this.push(copyBuffer(tap.buf, 0, pos));
}
var len = tap.pos - pos;
if (len > buf.length) {
// Not enough space for last written object, need to resize.
tap.buf = new Buffer(2 * len);
}
tap.pos = 0;
this._writeValue(tap, val); // Rewrite last failed write.
}
cb();
};
RawEncoder.prototype._flush = function (cb) {
var tap = this._tap;
var pos = tap.pos;
if (pos) {
// This should only ever be false if nothing is written to the stream.
this.push(tap.buf.slice(0, pos));
}
cb();
};
/**
* Duplex stream to write object container files.
*
* @param schema
* @param opts {Object}
*
* + `blockSize`, uncompressed.
* + `codec`
* + `codecs`
* + `noCheck`
* + `omitHeader`, useful to append to an existing block file.
*
*/
function BlockEncoder(schema, opts) {
opts = opts || {};
stream.Duplex.call(this, {
allowHalfOpen: true, // To support async compressors.
writableObjectMode: true
});
var obj, type;
if (types.Type.isType(schema)) {
type = schema;
schema = undefined;
} else {
// Keep full schema to be able to write it to the header later.
obj = files.load(schema);
type = types.createType(obj);
schema = JSON.stringify(obj);
}
this._schema = schema;
this._type = type;
this._writeValue = function (tap, val) {
try {
this._type._write(tap, val);
} catch (err) {
this.emit('error', err);
}
};
this._blockSize = opts.blockSize || 65536;
this._tap = new Tap(new Buffer(this._blockSize));
this._codecs = opts.codecs;
this._codec = opts.codec || 'null';
this._compress = null;
this._omitHeader = opts.omitHeader || false;
this._blockCount = 0;
this._syncMarker = opts.syncMarker || new utils.Lcg().nextBuffer(16);
this._queue = new utils.OrderedQueue();
this._pending = 0;
this._finished = false;
this._needPush = false;
this.on('finish', function () {
this._finished = true;
if (this._blockCount) {
this._flushChunk();
}
});
}
util.inherits(BlockEncoder, stream.Duplex);
BlockEncoder.getDefaultCodecs = function () {
return {
'null': function (buf, cb) { cb(null, buf); },
'deflate': zlib.deflateRaw
};
};
BlockEncoder.prototype._write = function (val, encoding, cb) {
var codec = this._codec;
this._compress = (this._codecs || BlockEncoder.getDefaultCodecs())[codec];
if (!this._compress) {
this.emit('error', new Error(f('unsupported codec: %s', codec)));
return;
}
if (!this._omitHeader) {
var meta = {
'avro.schema': new Buffer(this._schema || this._type.getSchema()),
'avro.codec': new Buffer(this._codec)
};
var Header = HEADER_TYPE.getRecordConstructor();
var header = new Header(MAGIC_BYTES, meta, this._syncMarker);
this.push(header.toBuffer());
}
this._write = this._writeChunk;
this._write(val, encoding, cb);
};
BlockEncoder.prototype._writeChunk = function (val, encoding, cb) {
var tap = this._tap;
var pos = tap.pos;
var flushing = false;
this._writeValue(tap, val);
if (!tap.isValid()) {
if (pos) {
this._flushChunk(pos, cb);
flushing = true;
}
var len = tap.pos - pos;
if (len > this._blockSize) {
// Not enough space for last written object, need to resize.
this._blockSize = len * 2;
}
tap.buf = new Buffer(this._blockSize);
tap.pos = 0;
this._writeValue(tap, val); // Rewrite last failed write.
}
this._blockCount++;
if (!flushing) {
cb();
}
};
BlockEncoder.prototype._flushChunk = function (pos, cb) {
var tap = this._tap;
pos = pos || tap.pos;
this._compress(tap.buf.slice(0, pos), this._createBlockCallback(cb));
this._blockCount = 0;
};
BlockEncoder.prototype._read = function () {
var self = this;
var data = this._queue.pop();
if (!data) {
if (this._finished && !this._pending) {
process.nextTick(function () { self.push(null); });
} else {
this._needPush = true;
}
return;
}
this.push(LONG_TYPE.toBuffer(data.count, true));
this.push(LONG_TYPE.toBuffer(data.buf.length, true));
this.push(data.buf);
this.push(this._syncMarker);
if (!this._finished) {
data.cb();
}
};
BlockEncoder.prototype._createBlockCallback = function (cb) {
var self = this;
var index = this._index++;
var count = this._blockCount;
this._pending++;
return function (err, data) {
if (err) {
self.emit('error', err);
return;
}
self._pending--;
self._queue.push(new BlockData(index, data, cb, count));
if (self._needPush) {
self._needPush = false;
self._read();
}
};
};
// Helpers.
/**
* An indexed block.
*
* This can be used to preserve block order since compression and decompression
* can cause some some blocks to be returned out of order. The count is only
* used when encoding.
*
*/
function BlockData(index, buf, cb, count) {
this.index = index;
this.buf = buf;
this.cb = cb;
this.count = count | 0;
}
/**
* Maybe get a block.
*
*/
function tryReadBlock(tap) {
var pos = tap.pos;
var block = BLOCK_TYPE._read(tap);
if (!tap.isValid()) {
tap.pos = pos;
return null;
}
return block;
}
/**
* Create bytes consumer, either reading or skipping records.
*
*/
function createReader(noDecode, type) {
if (noDecode) {
return (function (skipper) {
return function (tap) {
var pos = tap.pos;
skipper(tap);
return tap.buf.slice(pos, tap.pos);
};
})(type._skip);
} else {
return function (tap) { return type._read(tap); };
}
}
/**
* Copy a buffer.
*
* This avoids having to create a slice of the original buffer.
*
*/
function copyBuffer(buf, pos, len) {
var copy = new Buffer(len);
buf.copy(copy, 0, pos, pos + len);
return copy;
}
module.exports = {
HEADER_TYPE: HEADER_TYPE, // For tests.
MAGIC_BYTES: MAGIC_BYTES, // Idem.
streams: {
BlockDecoder: BlockDecoder,
BlockEncoder: BlockEncoder,
RawDecoder: RawDecoder,
RawEncoder: RawEncoder
}
};
/***/ },
/* 3 */
/***/ function(module, exports, __webpack_require__) {
/* jshint node: true */
'use strict';
/**
* Filesystem specifics.
*
* This module contains functions only used by node.js. It is shimmed by
* another module when `avsc` is required from `browserify`.
*
*/
var fs = __webpack_require__(4);
/**
* Try to load a schema.
*
* This method will attempt to load schemas from a file if the schema passed is
* a string which isn't valid JSON and contains at least one slash.
*
*/
function load(schema) {
var obj;
if (typeof schema == 'string' && schema !== 'null') {
// This last predicate is to allow `avro.parse('null')` to work similarly
// to `avro.parse('int')` and other primitives (null needs to be handled
// separately since it is also a valid JSON identifier).
try {
obj = JSON.parse(schema);
} catch (err) {
if (~schema.indexOf('/')) {
// This can't be a valid name, so we interpret is as a filepath. This
// makes is always feasible to read a file, independent of its name
// (i.e. even if its name is valid JSON), by prefixing it with `./`.
obj = JSON.parse(fs.readFileSync(schema));
}
}
}
if (obj === undefined) {
obj = schema;
}
return obj;
}
/**
* Default file loading function for assembling IDLs.
*
*/
function createImportHook() {
var imports = {};
return function (fpath, kind, cb) {
if (imports[fpath]) {
// Already imported, return nothing to avoid duplicating attributes.
process.nextTick(cb);
return;
}
imports[fpath] = true;
fs.readFile(fpath, {encoding: 'utf8'}, cb);
};
}
module.exports = {
createImportHook: createImportHook,
load: load
};
/***/ },
/* 4 */
/***/ function(module, exports) {
/***/ },
/* 5 */
/***/ function(module, exports, __webpack_require__) {
/* jshint node: true */
// TODO: Use `toFastProperties` on type reverse indices.
// TODO: Allow configuring when to write the size when writing arrays and maps,
// and customizing their block size.
// TODO: Code-generate `compare` and `clone` record and union methods.
'use strict';
/**
* This module defines all Avro data types and their serialization logic.
*
*/
var utils = __webpack_require__(6),
buffer = __webpack_require__(8), // For `SlowBuffer`.
util = __webpack_require__(9);
// Convenience imports.
var Tap = utils.Tap;
var f = util.format;
// All non-union concrete (i.e. non-logical) Avro types.
var TYPES = {
'array': ArrayType,
'boolean': BooleanType,
'bytes': BytesType,
'double': DoubleType,
'enum': EnumType,
'error': RecordType,
'fixed': FixedType,
'float': FloatType,
'int': IntType,
'long': LongType,
'map': MapType,
'null': NullType,
'record': RecordType,
'string': StringType
};
// Valid (field, type, and symbol) name regex.
var NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
// Random generator.
var RANDOM = new utils.Lcg();
// Encoding tap (shared for performance).
var TAP = new Tap(new buffer.SlowBuffer(1024));
// Currently active logical type, used for name redirection.
var LOGICAL_TYPE = null;
// Variable used to decide whether to include logical attributes when getting a
// type's schema. A global variable is the simplest way to "pass an argument"
// to JSON stringify's replacer function.
var EXPORT_ATTRS = false;
/**
* Schema parsing entry point.
*
* It isn't exposed directly but called from `parse` inside `index.js` (node)
* or `avsc.js` (browserify) which each add convenience functionality.
*
*/
function createType(attrs, opts) {
if (attrs === null) {
// Let's be helpful for this common error.
throw new Error('invalid type: null (did you mean "null"?)');
}
if (Type.isType(attrs)) {
return attrs;
}
opts = opts || {};
opts.registry = opts.registry || {};
var type;
if (typeof attrs == 'string') { // Type reference.
attrs = qualify(attrs, opts.namespace);
type = opts.registry[attrs];
if (type) {
// Type was already defined, return it.
return type;
}
if (isPrimitive(attrs)) {
// Reference to a primitive type. These are also defined names by default
// so we create the appropriate type and it to the registry for future
// reference.
return opts.registry[attrs] = createType({type: attrs}, opts);
}
throw new Error(f('undefined type name: %s', attrs));
}
if (opts.typeHook && (type = opts.typeHook(attrs, opts))) {
if (!Type.isType(type)) {
throw new Error(f('invalid typehook return value: %j', type));
}
return type;
}
if (attrs.logicalType && opts.logicalTypes && !LOGICAL_TYPE) {
var DerivedType = opts.logicalTypes[attrs.logicalType];
if (DerivedType) {
var namespace = opts.namespace;
var registry = {};
Object.keys(opts.registry).forEach(function (key) {
registry[key] = opts.registry[key];
});
try {
return new DerivedType(attrs, opts);
} catch (err) {
if (opts.assertLogicalTypes) {
// The spec mandates that we fall through to the underlying type if
// the logical type is invalid. We provide this option to ease
// debugging.
throw err;
}
LOGICAL_TYPE = null;
opts.namespace = namespace;
opts.registry = registry;
}
}
}
if (Array.isArray(attrs)) { // Union.
var UnionType = opts.wrapUnions ? WrappedUnionType : UnwrappedUnionType;
type = new UnionType(attrs, opts);
} else { // New type definition.
type = (function (typeName) {
var Type = TYPES[typeName];
if (Type === undefined) {
throw new Error(f('unknown type: %j', typeName));
}
return new Type(attrs, opts);
})(attrs.type);
}
return type;
}
/**
* "Abstract" base Avro type.
*
* This class' constructor will register any named types to support recursive
* schemas. All type values are represented in memory similarly to their JSON
* representation, except for:
*
* + `bytes` and `fixed` which are represented as `Buffer`s.
* + `union`s which will be "unwrapped" unless the `wrapUnions` option is set.
*
* See individual subclasses for details.
*
*/
function Type(attrs, opts) {
var type = LOGICAL_TYPE || this;
LOGICAL_TYPE = null;
// Lazily instantiated hash string. It will be generated the first time the
// type's default fingerprint is computed (for example when using `equals`).
this._hs = undefined;
this._name = undefined;
this._aliases = undefined;
if (attrs) {
// This is a complex (i.e. non-primitive) type.
var name = attrs.name;
var namespace = attrs.namespace === undefined ?
opts && opts.namespace :
attrs.namespace;
if (name !== undefined) {
// This isn't an anonymous type.
name = qualify(name, namespace);
if (isPrimitive(name)) {
// Avro doesn't allow redefining primitive names.
throw new Error(f('cannot rename primitive type: %j', name));
}
var registry = opts && opts.registry;
if (registry) {
if (registry[name] !== undefined) {
throw new Error(f('duplicate type name: %s', name));
}
registry[name] = type;
}
} else if (opts && opts.noAnonymousTypes) {
throw new Error(f('missing name property in schema: %j', attrs));
}
this._name = name;
this._aliases = attrs.aliases ?
attrs.aliases.map(function (s) { return qualify(s, namespace); }) :
[];
}
}
Type.isType = function (/* any, [prefix] ... */) {
var l = arguments.length;
if (!l) {
return false;
}
var any = arguments[0];
if (
!any ||
typeof any._update != 'function' ||
typeof any.getTypeName != 'function'
) {
// Not fool-proof, but most likely good enough.
return false;
}
if (l === 1) {
// No type names specified, we are done.
return true;
}
// We check if at least one of the prefixes matches.
var typeName = any.getTypeName();
var i;
for (i = 1; i < l; i++) {
if (typeName.indexOf(arguments[i]) === 0) {
return true;
}
}
return false;
};
Type.__reset = function (size) { TAP.buf = new buffer.SlowBuffer(size); };
Type.prototype.createResolver = function (type, opts) {
if (!Type.isType(type)) {
// More explicit error message than the "incompatible type" thrown
// otherwise (especially because of the overridden `toJSON` method).
throw new Error(f('not a type: %j', type));
}
if (!Type.isType(this, 'logical') && Type.isType(type, 'logical')) {
// Trying to read a logical type as a built-in: unwrap the logical type.
return this.createResolver(type._underlyingType, opts);
}
opts = opts || {};
opts.registry = opts.registry || {};
var resolver, key;
if (
Type.isType(this, 'record', 'error') &&
Type.isType(type, 'record', 'error')
) {
// We allow conversions between records and errors.
key = this._name + ':' + type._name; // ':' is illegal in Avro type names.
resolver = opts.registry[key];
if (resolver) {
return resolver;
}
}
resolver = new Resolver(this);
if (key) { // Register resolver early for recursive schemas.
opts.registry[key] = resolver;
}
if (Type.isType(type, 'union')) {
var resolvers = type._types.map(function (t) {
return this.createResolver(t, opts);
}, this);
resolver._read = function (tap) {
var index = tap.readLong();
var resolver = resolvers[index];
if (resolver === undefined) {
throw new Error(f('invalid union index: %s', index));
}
return resolvers[index]._read(tap);
};
} else {
this._update(resolver, type, opts);
}
if (!resolver._read) {
throw new Error(f('cannot read %s as %s', type, this));
}
return resolver;
};
Type.prototype.decode = function (buf, pos, resolver) {
var tap = new Tap(buf, pos);
var val = readValue(this, tap, resolver);
if (!tap.isValid()) {
return {value: undefined, offset: -1};
}
return {value: val, offset: tap.pos};
};
Type.prototype.encode = function (val, buf, pos) {
var tap = new Tap(buf, pos);
this._write(tap, val);
if (!tap.isValid()) {
// Don't throw as there is no way to predict this. We also return the
// number of missing bytes to ease resizing.
return buf.length - tap.pos;
}
return tap.pos;
};
Type.prototype.fromBuffer = function (buf, resolver, noCheck) {
var tap = new Tap(buf);
var val = readValue(this, tap, resolver, noCheck);
if (!tap.isValid()) {
throw new Error('truncated buffer');
}
if (!noCheck && tap.pos < buf.length) {
throw new Error('trailing data');
}
return val;
};
Type.prototype.toBuffer = function (val) {
TAP.pos = 0;
this._write(TAP, val);
if (!TAP.isValid()) {
Type.__reset(2 * TAP.pos);
TAP.pos = 0;
this._write(TAP, val);
}
var buf = new Buffer(TAP.pos);
TAP.buf.copy(buf, 0, 0, TAP.pos);
return buf;
};
Type.prototype.fromString = function (str) {
return this._copy(JSON.parse(str), {coerce: 2});
};
Type.prototype.toString = function (val) {
if (val === undefined) {
// Consistent behavior with standard `toString` expectations.
return this.getSchema({noDeref: true});
}
return JSON.stringify(this._copy(val, {coerce: 3}));
};
Type.prototype.clone = function (val, opts) {
if (opts) {
opts = {
coerce: !!opts.coerceBuffers | 0, // Coerce JSON to Buffer.
fieldHook: opts.fieldHook,
qualifyNames: !!opts.qualifyNames,
wrap: !!opts.wrapUnions | 0 // Wrap first match into union.
};
return this._copy(val, opts);
} else {
// If no modifications are required, we can get by with a serialization
// roundtrip (generally much faster than a standard deep copy).
return this.fromBuffer(this.toBuffer(val));
}
};
Type.prototype.isValid = function (val, opts) {
// We only have a single flag for now, so no need to complicate things.
var flags = (opts && opts.noUndeclaredFields) | 0;
var errorHook = opts && opts.errorHook;
var hook, path;
if (errorHook) {
path = [];
hook = function (any, type) {
errorHook.call(this, path.slice(), any, type, val);
};
}
return this._check(val, flags, hook, path);
};
Type.prototype.compareBuffers = function (buf1, buf2) {
return this._match(new Tap(buf1), new Tap(buf2));
};
Type.prototype.getName = function (asBranch) {
var type = Type.isType(this, 'logical') ? this._underlyingType : this;
if (type._name || !asBranch) {
return type._name;
}
return Type.isType(this, 'union') ? undefined : type.getTypeName();
};
Type.prototype.getSchema = function (opts) { return stringify(this, opts); };
Type.prototype.equals = function (type) {
return (
Type.isType(type) &&
this.getFingerprint().equals(type.getFingerprint())
);
};
Type.prototype.getFingerprint = function (algorithm) {
if (!algorithm) {
if (!this._hs) {
this._hs = utils.getHash(this.getSchema()).toString('binary');
}
return new Buffer(this._hs, 'binary');
} else {
return utils.getHash(this.getSchema(), algorithm);
}
};
Type.prototype.inspect = function () {
var typeName = this.getTypeName();
var className = getClassName(typeName);
if (isPrimitive(typeName)) {
// The class name is sufficient to identify the type.
return f('<%s>', className);
} else {
// We add a little metadata for convenience.
var obj = JSON.parse(this.getSchema({exportAttrs: true, noDeref: true}));
if (typeof obj == 'object' && !Type.isType(this, 'logical')) {
obj.type = undefined; // Would be redundant with constructor name.
}
return f('<%s %j>', className, obj);
}
};
Type.prototype._check = utils.abstractFunction;
Type.prototype._copy = utils.abstractFunction;
Type.prototype._match = utils.abstractFunction;
Type.prototype._read = utils.abstractFunction;
Type.prototype._skip = utils.abstractFunction;
Type.prototype._update = utils.abstractFunction;
Type.prototype._write = utils.abstractFunction;
Type.prototype.compare = utils.abstractFunction;
Type.prototype.getTypeName = utils.abstractFunction;
Type.prototype.random = utils.abstractFunction;
// Implementations.
/**
* Base primitive Avro type.
*
* Most of the primitive types share the same cloning and resolution
* mechanisms, provided by this class. This class also lets us conveniently
* check whether a type is a primitive using `instanceof`.
*
*/
function PrimitiveType() { Type.call(this); }
util.inherits(PrimitiveType, Type);
PrimitiveType.prototype._update = function (resolver, type) {
if (type.constructor === this.constructor) {
resolver._read = this._read;
}
};
PrimitiveType.prototype._copy = function (val) {
this._check(val, undefined, throwInvalidError);
return val;
};
PrimitiveType.prototype.compare = utils.compare;
PrimitiveType.prototype.toJSON = function () { return this.getTypeName(); };
/**
* Nulls.
*
*/
function NullType() { PrimitiveType.call(this); }
util.inherits(NullType, PrimitiveType);
NullType.prototype._check = function (val, flags, hook) {
var b = val === null;
if (!b && hook) {
hook(val, this);
}
return b;
};
NullType.prototype._read = function () { return null; };
NullType.prototype._skip = function () {};
NullType.prototype._write = function (tap, val) {
if (val !== null) {
throwInvalidError(val, this);
}
};
NullType.prototype._match = function () { return 0; };
NullType.prototype.compare = NullType.prototype._match;
NullType.prototype.getTypeName = function () { return 'null'; };
NullType.prototype.random = NullType.prototype._read;
/**
* Booleans.
*
*/
function BooleanType() { PrimitiveType.call(this); }
util.inherits(BooleanType, PrimitiveType);
BooleanType.prototype._check = function (val, flags, hook) {
var b = typeof val == 'boolean';
if (!b && hook) {
hook(val, this);
}
return b;
};
BooleanType.prototype._read = function (tap) { return tap.readBoolean(); };
BooleanType.prototype._skip = function (tap) { tap.skipBoolean(); };
BooleanType.prototype._write = function (tap, val) {
if (typeof val != 'boolean') {
throwInvalidError(val, this);
}
tap.writeBoolean(val);
};
BooleanType.prototype._match = function (tap1, tap2) {
return tap1.matchBoolean(tap2);
};
BooleanType.prototype.getTypeName = function () { return 'boolean'; };
BooleanType.prototype.random = function () { return RANDOM.nextBoolean(); };
/**
* Integers.
*
*/
function IntType() { PrimitiveType.call(this); }
util.inherits(IntType, PrimitiveType);
IntType.prototype._check = function (val, flags, hook) {
var b = val === (val | 0);
if (!b && hook) {
hook(val, this);
}
return b;
};
IntType.prototype._read = function (tap) { return tap.readInt(); };
IntType.prototype._skip = function (tap) { tap.skipInt(); };
IntType.prototype._write = function (tap, val) {
if (val !== (val | 0)) {
throwInvalidError(val, this);
}
tap.writeInt(val);
};
IntType.prototype._match = function (tap1, tap2) {
return tap1.matchInt(tap2);
};
IntType.prototype.getTypeName = function () { return 'int'; };
IntType.prototype.random = function () { return RANDOM.nextInt(1000) | 0; };
/**
* Longs.
*
* We can't capture all the range unfortunately since JavaScript represents all
* numbers internally as `double`s, so the default implementation plays safe
* and throws rather than potentially silently change the data. See `__with` or
* `AbstractLongType` below for a way to implement a custom long type.
*
*/
function LongType() { PrimitiveType.call(this); }
util.inherits(LongType, PrimitiveType);
LongType.prototype._check = function (val, flags, hook) {
var b = typeof val == 'number' && val % 1 === 0 && isSafeLong(val);
if (!b && hook) {
hook(val, this);
}
return b;
};
LongType.prototype._read = function (tap) {
var n = tap.readLong();
if (!isSafeLong(n)) {
throw new Error('potential precision loss');
}
return n;
};
LongType.prototype._skip = function (tap) { tap.skipLong(); };
LongType.prototype._write = function (tap, val) {
if (typeof val != 'number' || val % 1 || !isSafeLong(val)) {
throwInvalidError(val, this);
}
tap.writeLong(val);
};
LongType.prototype._match = function (tap1, tap2) {
return tap1.matchLong(tap2);
};
LongType.prototype._update = function (resolver, type) {
switch (type.getTypeName()) {
case 'int':
case 'long':
resolver._read = type._read;
}
};
LongType.prototype.getTypeName = function () { return 'long'; };
LongType.prototype.random = function () { return RANDOM.nextInt(); };
LongType.__with = function (methods, noUnpack) {
methods = methods || {}; // Will give a more helpful error message.
// We map some of the methods to a different name to be able to intercept
// their input and output (otherwise we wouldn't be able to perform any
// unpacking logic, and the type wouldn't work when nested).
var mapping = {
toBuffer: '_toBuffer',
fromBuffer: '_fromBuffer',
fromJSON: '_fromJSON',
toJSON: '_toJSON',
isValid: '_isValid',
compare: 'compare'
};
var type = new AbstractLongType(noUnpack);
Object.keys(mapping).forEach(function (name) {
if (methods[name] === undefined) {
throw new Error(f('missing method implementation: %s', name));
}
type[mapping[name]] = methods[name];
});
return type;
};
/**
* Floats.
*
*/
function FloatType() { PrimitiveType.call(this); }
util.inherits(FloatType, PrimitiveType);
FloatType.prototype._check = function (val, flags, hook) {
var b = typeof val == 'number';
if (!b && hook) {
hook(val, this);
}
return b;
};
FloatType.prototype._read = function (tap) { return tap.readFloat(); };
FloatType.prototype._skip = function (tap) { tap.skipFloat(); };
FloatType.prototype._write = function (tap, val) {
if (typeof val != 'number') {
throwInvalidError(val, this);
}
tap.writeFloat(val);
};
FloatType.prototype._match = function (tap1, tap2) {
return tap1.matchFloat(tap2);
};
FloatType.prototype._update = function (resolver, type) {
switch (type.getTypeName()) {
case 'float':
case 'int':
case 'long':
resolver._read = type._read;
}
};
FloatType.prototype.getTypeName = function () { return 'float'; };
FloatType.prototype.random = function () { return RANDOM.nextFloat(1e3); };
/**
* Doubles.
*
*/
function DoubleType() { PrimitiveType.call(this); }
util.inherits(DoubleType, PrimitiveType);
DoubleType.prototype._check = function (val, flags, hook) {
var b = typeof val == 'number';
if (!b && hook) {
hook(val, this);
}
return b;
};
DoubleType.prototype._read = function (tap) { return tap.readDouble(); };
DoubleType.prototype._skip = function (tap) { tap.skipDouble(); };
DoubleType.prototype._write = function (tap, val) {
if (typeof val != 'number') {
throwInvalidError(val, this);
}
tap.writeDouble(val);
};
DoubleType.prototype._match = function (tap1, tap2) {
return tap1.matchDouble(tap2);
};
DoubleType.prototype._update = function (resolver, type) {
switch (type.getTypeName()) {
case 'double':
case 'float':
case 'int':
case 'long':
resolver._read = type._read;
}
};
DoubleType.prototype.getTypeName = function () { return 'double'; };
DoubleType.prototype.random = function () { return RANDOM.nextFloat(); };
/**
* Strings.
*
*/
function StringType() { PrimitiveType.call(this); }
util.inherits(StringType, PrimitiveType);
StringType.prototype._check = function (val, flags, hook) {
var b = typeof val == 'string';
if (!b && hook) {
hook(val, this);
}
return b;
};
StringType.prototype._read = function (tap) { return tap.readString(); };
StringType.prototype._skip = function (tap) { tap.skipString(); };
StringType.prototype._write = function (tap, val) {
if (typeof val != 'string') {
throwInvalidError(val, this);
}
tap.writeString(val);
};
StringType.prototype._match = function (tap1, tap2) {
return tap1.matchString(tap2);
};
StringType.prototype._update = function (resolver, type) {
switch (type.getTypeName()) {
case 'bytes':
case 'string':
resolver._read = this._read;
}
};
StringType.prototype.getTypeName = function () { return 'string'; };
StringType.prototype.random = function () {
return RANDOM.nextString(RANDOM.nextInt(32));
};
/**
* Bytes.
*
* These are represented in memory as `Buffer`s rather than binary-encoded
* strings. This is more efficient (when decoding/encoding from bytes, the
* common use-case), idiomatic, and convenient.
*
* Note the coercion in `_copy`.
*
*/
function BytesType() { PrimitiveType.call(this); }
util.inherits(BytesType, PrimitiveType);
BytesType.prototype._check = function (val, flags, hook) {
var b = Buffer.isBuffer(val);
if (!b && hook) {
hook(val, this);
}
return b;
};
BytesType.prototype._read = function (tap) { return tap.readBytes(); };
BytesType.prototype._skip = function (tap) { tap.skipBytes(); };
BytesType.prototype._write = function (tap, val) {
if (!Buffer.isBuffer(val)) {
throwInvalidError(val, this);
}
tap.writeBytes(val);
};
BytesType.prototype._match = function (tap1, tap2) {
return tap1.matchBytes(tap2);
};
BytesType.prototype._update = StringType.prototype._update;
BytesType.prototype._copy = function (obj, opts) {
var buf;
switch ((opts && opts.coerce) | 0) {
case 3: // Coerce buffers to strings.
this._check(obj, undefined, throwInvalidError);
return obj.toString('binary');
case 2: // Coerce strings to buffers.
if (typeof obj != 'string') {
throw new Error(f('cannot coerce to buffer: %j', obj));
}
buf = new Buffer(obj, 'binary');
this._check(buf, undefined, throwInvalidError);
return buf;
case 1: // Coerce buffer JSON representation to buffers.
if (!isJsonBuffer(obj)) {
throw new Error(f('cannot coerce to buffer: %j', obj));
}
buf = new Buffer(obj.data);
this._check(buf, undefined, throwInvalidError);
return buf;
default: // Copy buffer.
this._check(obj, undefined, throwInvalidError);
return new Buffer(obj);
}
};
BytesType.prototype.compare = Buffer.compare;
BytesType.prototype.getTypeName = function () { return 'bytes'; };
BytesType.prototype.random = function () {
return RANDOM.nextBuffer(RANDOM.nextInt(32));
};
/**
* Base "abstract" Avro union type.
*
*/
function UnionType(attrs, opts) {
Type.call(this);
if (!Array.isArray(attrs)) {
throw new Error(f('non-array union schema: %j', attrs));
}
if (!attrs.length) {
throw new Error('empty union');
}
this._types = attrs.map(function (obj) { return createType(obj, opts); });
this._branchIndices = {};
this._types.forEach(function (type, i) {
if (Type.isType(type, 'union')) {
throw new Error('unions cannot be directly nested');
}
var branch = type.getName(true);
if (this._branchIndices[branch] !== undefined) {
throw new Error(f('duplicate union branch name: %j', branch));
}
this._branchIndices[branch] = i;
}, this);
}
util.inherits(UnionType, Type);
UnionType.prototype._skip = function (tap) {
this._types[tap.readLong()]._skip(tap);
};
UnionType.prototype._match = function (tap1, tap2) {
var n1 = tap1.readLong();
var n2 = tap2.readLong();
if (n1 === n2) {
return this._types[n1]._match(tap1, tap2);
} else {
return n1 < n2 ? -1 : 1;
}
};
UnionType.prototype.getTypes = function () { return this._types.slice(); };
UnionType.prototype.toJSON = function () { return this._types; };
/**
* "Natural" union type.
*
* This representation doesn't require a wrapping object and is therefore
* simpler and generally closer to what users expect. However it cannot be used
* to represent all Avro unions since some lead to ambiguities (e.g. if two
* number types are in the union).
*
* Currently, this union supports at most one type in each of the categories
* below:
*
* + `null`
* + `boolean`
* + `int`, `long`, `float`, `double`
* + `string`, `enum`
* + `bytes`, `fixed`
* + `array`
* + `map`, `record`
*
*/
function UnwrappedUnionType(attrs, opts) {
UnionType.call(this, attrs, opts);
this._logicalBranches = null;
this._bucketIndices = {};
this._types.forEach(function (type, index) {
if (Type.isType(type, 'logical')) {
if (!this._logicalBranches) {
this._logicalBranches = [];
}
this._logicalBranches.push({index: index, type: type});
} else {
var bucket = getTypeBucket(type);
if (this._bucketIndices[bucket] !== undefined) {
throw new Error(f('ambiguous unwrapped union: %j', this));
}
this._bucketIndices[bucket] = index;
}
}, this);
}
util.inherits(UnwrappedUnionType, UnionType);
UnwrappedUnionType.prototype._getIndex = function (val) {
var index = this._bucketIndices[getValueBucket(val)];
if (this._logicalBranches) {
// Slower path, we must run the value through all logical types.
index = this._getLogicalIndex(val, index);
}
return index;
};
UnwrappedUnionType.prototype._getLogicalIndex = function (any, index) {
var logicalBranches = this._logicalBranches;
var i, l, branch;
for (i = 0, l = logicalBranches.length; i < l; i++) {
branch = logicalBranches[i];
if (branch.type._check(any)) {
if (index === undefined) {
index = branch.index;
} else {
// More than one branch matches the value so we aren't guaranteed to
// infer the correct type. We throw rather than corrupt data. This can
// be fixed by "tightening" the logical types.
throw new Error('ambiguous conversion');
}
}
}
return index;
};
UnwrappedUnionType.prototype._check = function (val, flags, hook, path) {
var index = this._getIndex(val);
var b = index !== undefined;
if (b) {
return this._types[index]._check(val, flags, hook, path);
}
if (hook) {
hook(val, this);
}
return b;
};
UnwrappedUnionType.prototype._read = function (tap) {
var index = tap.readLong();
var branchType = this._types[index];
if (branchType) {
return branchType._read(tap);
} else {
throw new Error(f('invalid union index: %s', index));
}
};
UnwrappedUnionType.prototype._write = function (tap, val) {
var index = this._getIndex(val);
if (index === undefined) {
throwInvalidError(val, this);
}
tap.writeLong(index);
if (val !== null) {
this._types[index]._write(tap, val);
}
};
UnwrappedUnionType.prototype._update = function (resolver, type, opts) {
// jshint -W083
// (The loop exits after the first function is created.)
var i, l, typeResolver;
for (i = 0, l = this._types.length; i < l; i++) {
try {
typeResolver = this._types[i].createResolver(type, opts);
} catch (err) {
continue;
}
resolver._read = function (tap) { return typeResolver._read(tap); };
return;
}
};
UnwrappedUnionType.prototype._copy = function (val, opts) {
var coerce = opts && opts.coerce | 0;
var wrap = opts && opts.wrap | 0;
var index;
if (wrap === 2) {
// We are parsing a default, so alw