UNPKG

react-native-avsc

Version:
1,873 lines (1,601 loc) 514 kB
(function webpackUniversalModuleDefinition(root, factory) { if(typeof exports === 'object' && typeof module === 'object') module.exports = factory(require("buffer"), require("stream-browserify")); else if(typeof define === 'function' && define.amd) define(["buffer", "stream-browserify"], factory); else if(typeof exports === 'object') exports["avsc"] = factory(require("buffer"), require("stream-browserify")); else root["avsc"] = factory(root["buffer"], root["stream-browserify"]); })(this, function(__WEBPACK_EXTERNAL_MODULE_8__, __WEBPACK_EXTERNAL_MODULE_12__) { return /******/ (function(modules) { // webpackBootstrap /******/ // The module cache /******/ var installedModules = {}; /******/ // The require function /******/ function __webpack_require__(moduleId) { /******/ // Check if module is in cache /******/ if(installedModules[moduleId]) /******/ return installedModules[moduleId].exports; /******/ // Create a new module (and put it into the cache) /******/ var module = installedModules[moduleId] = { /******/ exports: {}, /******/ id: moduleId, /******/ loaded: false /******/ }; /******/ // Execute the module function /******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); /******/ // Flag the module as loaded /******/ module.loaded = true; /******/ // Return the exports of the module /******/ return module.exports; /******/ } /******/ // expose the modules object (__webpack_modules__) /******/ __webpack_require__.m = modules; /******/ // expose the module cache /******/ __webpack_require__.c = installedModules; /******/ // __webpack_public_path__ /******/ __webpack_require__.p = ""; /******/ // Load entry module and return exports /******/ return __webpack_require__(0); /******/ }) /************************************************************************/ /******/ ([ /* 0 */ /***/ function(module, exports, __webpack_require__) { // Get original avsc var avscWrapper = __webpack_require__(1); // Delete decodeFile because we don't want to read files in browser delete avscWrapper.decodeFile; // Delete getFileHeader because we don't want to read files in browser delete avscWrapper.getFileHeader; // Export module.exports = avscWrapper; /***/ }, /* 1 */ /***/ function(module, exports, __webpack_require__) { /* jshint node: true */ 'use strict'; /** * Node.js entry point (see `etc/browser/` for browserify's entry points). * * It also adds Node.js specific functionality (for example a few convenience * functions to read Avro files from the local filesystem). * */ var containers = __webpack_require__(2), files = __webpack_require__(3), protocols = __webpack_require__(38), schemas = __webpack_require__(40), types = __webpack_require__(5), utils = __webpack_require__(6), values = __webpack_require__(42), fs = __webpack_require__(4); /** * Parse a schema and return the corresponding type or protocol. * */ function parse(schema, opts) { var attrs = files.load(schema); return attrs.protocol ? protocols.createProtocol(attrs, opts) : types.createType(attrs, opts); } /** * Extract a container file's header synchronously. * */ function extractFileHeader(path, opts) { opts = opts || {}; var decode = opts.decode === undefined ? true : !!opts.decode; var size = Math.max(opts.size || 4096, 4); var fd = fs.openSync(path, 'r'); var buf = new Buffer(size); var pos = 0; var tap = new utils.Tap(buf); var header = null; while (pos < 4) { // Make sure we have enough to check the magic bytes. pos += fs.readSync(fd, buf, pos, size - pos); } if (containers.MAGIC_BYTES.equals(buf.slice(0, 4))) { do { header = containers.HEADER_TYPE._read(tap); } while (!isValid()); if (decode !== false) { var meta = header.meta; meta['avro.schema'] = JSON.parse(meta['avro.schema'].toString()); if (meta['avro.codec'] !== undefined) { meta['avro.codec'] = meta['avro.codec'].toString(); } } } fs.closeSync(fd); return header; function isValid() { if (tap.isValid()) { return true; } var len = 2 * tap.buf.length; var buf = new Buffer(len); len = fs.readSync(fd, buf, 0, len); tap.buf = Buffer.concat([tap.buf, buf]); tap.pos = 0; return false; } } /** * Readable stream of records from a local Avro file. * */ function createFileDecoder(path, opts) { return fs.createReadStream(path) .pipe(new containers.streams.BlockDecoder(opts)); } /** * Writable stream of records to a local Avro file. * */ function createFileEncoder(path, schema, opts) { var encoder = new containers.streams.BlockEncoder(schema, opts); encoder.pipe(fs.createWriteStream(path, {defaultEncoding: 'binary'})); return encoder; } module.exports = { Protocol: protocols.Protocol, Type: types.Type, assemble: schemas.assemble, combine: values.combine, createFileDecoder: createFileDecoder, createFileEncoder: createFileEncoder, extractFileHeader: extractFileHeader, infer: values.infer, parse: parse, streams: containers.streams, types: types.builtins }; /***/ }, /* 2 */ /***/ function(module, exports, __webpack_require__) { /* jshint node: true */ // TODO: Add streams which prefix each record with its length. 'use strict'; /** * This module defines custom streams to write and read Avro files. * * In particular, the `Block{En,De}coder` streams are able to deal with Avro * container files. None of the streams below depend on the filesystem however, * this way they can also be used in the browser (for example to parse HTTP * responses). * */ var files = __webpack_require__(3), types = __webpack_require__(5), utils = __webpack_require__(6), stream = __webpack_require__(12), util = __webpack_require__(9), zlib = __webpack_require__(13); // Type of Avro header. var HEADER_TYPE = types.createType({ namespace: 'org.apache.avro.file', name: 'Header', type: 'record', fields : [ {name: 'magic', type: {type: 'fixed', name: 'Magic', size: 4}}, {name: 'meta', type: {type: 'map', values: 'bytes'}}, {name: 'sync', type: {type: 'fixed', name: 'Sync', size: 16}} ] }); // Type of each block. var BLOCK_TYPE = types.createType({ namespace: 'org.apache.avro.file', name: 'Block', type: 'record', fields : [ {name: 'count', type: 'long'}, {name: 'data', type: 'bytes'}, {name: 'sync', type: {type: 'fixed', name: 'Sync', size: 16}} ] }); // Used to toBuffer each block, without having to copy all its data. var LONG_TYPE = types.createType('long'); // First 4 bytes of an Avro object container file. var MAGIC_BYTES = new Buffer('Obj\x01'); // Convenience. var f = util.format; var Tap = utils.Tap; /** * Duplex stream for decoding fragments. * */ function RawDecoder(schema, opts) { opts = opts || {}; var noDecode = !!opts.noDecode; stream.Duplex.call(this, { readableObjectMode: !noDecode, allowHalfOpen: false }); this._type = types.createType(files.load(schema)); this._tap = new Tap(new Buffer(0)); this._writeCb = null; this._needPush = false; this._readValue = createReader(noDecode, this._type); this._finished = false; this.on('finish', function () { this._finished = true; this._read(); }); } util.inherits(RawDecoder, stream.Duplex); RawDecoder.prototype._write = function (chunk, encoding, cb) { // Store the write callback and call it when we are done decoding all records // in this chunk. If we call it right away, we risk loading the entire input // in memory. We only need to store the latest callback since the stream API // guarantees that `_write` won't be called again until we call the previous. this._writeCb = cb; var tap = this._tap; tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]); tap.pos = 0; if (this._needPush) { this._needPush = false; this._read(); } }; RawDecoder.prototype._read = function () { this._needPush = false; var tap = this._tap; var pos = tap.pos; var val = this._readValue(tap); if (tap.isValid()) { this.push(val); } else if (!this._finished) { tap.pos = pos; this._needPush = true; if (this._writeCb) { // This should only ever be false on the first read, and only if it // happens before the first write. this._writeCb(); } } else { this.push(null); } }; /** * Duplex stream for decoding object container files. * */ function BlockDecoder(opts) { opts = opts || {}; var noDecode = !!opts.noDecode; stream.Duplex.call(this, { allowHalfOpen: true, // For async decompressors. readableObjectMode: !noDecode }); this._type = null; this._codecs = opts.codecs; this._parseHook = opts.parseHook; this._tap = new Tap(new Buffer(0)); this._blockTap = new Tap(new Buffer(0)); this._syncMarker = null; this._readValue = null; this._noDecode = noDecode; this._queue = new utils.OrderedQueue(); this._decompress = null; // Decompression function. this._index = 0; // Next block index. this._needPush = false; this._finished = false; this.on('finish', function () { this._finished = true; if (this._needPush) { this._read(); } }); } util.inherits(BlockDecoder, stream.Duplex); BlockDecoder.getDefaultCodecs = function () { return { 'null': function (buf, cb) { cb(null, buf); }, 'deflate': zlib.inflateRaw }; }; BlockDecoder.prototype._decodeHeader = function () { var tap = this._tap; if (tap.buf.length < MAGIC_BYTES.length) { // Wait until more data arrives. return false; } if (!MAGIC_BYTES.equals(tap.buf.slice(0, MAGIC_BYTES.length))) { this.emit('error', new Error('invalid magic bytes')); return false; } var header = HEADER_TYPE._read(tap); if (!tap.isValid()) { return false; } var codec = (header.meta['avro.codec'] || 'null').toString(); this._decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec]; if (!this._decompress) { this.emit('error', new Error(f('unknown codec: %s', codec))); return; } try { var schema = JSON.parse(header.meta['avro.schema'].toString()); if (this._parseHook) { schema = this._parseHook(schema); } this._type = types.createType(schema); } catch (err) { this.emit('error', err); return; } this._readValue = createReader(this._noDecode, this._type); this._syncMarker = header.sync; this.emit('metadata', this._type, codec, header); return true; }; BlockDecoder.prototype._write = function (chunk, encoding, cb) { var tap = this._tap; tap.buf = Buffer.concat([tap.buf, chunk]); tap.pos = 0; if (!this._decodeHeader()) { process.nextTick(cb); return; } // We got the header, switch to block decoding mode. Also, call it directly // in case we already have all the data (in which case `_write` wouldn't get // called anymore). this._write = this._writeChunk; this._write(new Buffer(0), encoding, cb); }; BlockDecoder.prototype._writeChunk = function (chunk, encoding, cb) { var tap = this._tap; tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]); tap.pos = 0; var nBlocks = 1; var block; while ((block = tryReadBlock(tap))) { if (!this._syncMarker.equals(block.sync)) { this.emit('error', new Error('invalid sync marker')); return; } nBlocks++; this._decompress(block.data, this._createBlockCallback(chunkCb)); } chunkCb(); function chunkCb() { if (!--nBlocks) { cb(); } } }; BlockDecoder.prototype._createBlockCallback = function (cb) { var self = this; var index = this._index++; return function (err, data) { if (err) { self.emit('error', err); cb(); } else { self._queue.push(new BlockData(index, data, cb)); if (self._needPush) { self._read(); } } }; }; BlockDecoder.prototype._read = function () { this._needPush = false; var tap = this._blockTap; if (tap.pos >= tap.buf.length) { var data = this._queue.pop(); if (!data) { if (this._finished) { this.push(null); } else { this._needPush = true; } return; // Wait for more data. } data.cb(); tap.buf = data.buf; tap.pos = 0; } this.push(this._readValue(tap)); // The read is guaranteed valid. }; /** * Duplex stream for encoding. * */ function RawEncoder(schema, opts) { opts = opts || {}; stream.Transform.call(this, { writableObjectMode: true, allowHalfOpen: false }); this._type = types.createType(files.load(schema)); this._writeValue = function (tap, val) { try { this._type._write(tap, val); } catch (err) { this.emit('error', err); } }; this._tap = new Tap(new Buffer(opts.batchSize || 65536)); } util.inherits(RawEncoder, stream.Transform); RawEncoder.prototype._transform = function (val, encoding, cb) { var tap = this._tap; var buf = tap.buf; var pos = tap.pos; this._writeValue(tap, val); if (!tap.isValid()) { if (pos) { // Emit any valid data. this.push(copyBuffer(tap.buf, 0, pos)); } var len = tap.pos - pos; if (len > buf.length) { // Not enough space for last written object, need to resize. tap.buf = new Buffer(2 * len); } tap.pos = 0; this._writeValue(tap, val); // Rewrite last failed write. } cb(); }; RawEncoder.prototype._flush = function (cb) { var tap = this._tap; var pos = tap.pos; if (pos) { // This should only ever be false if nothing is written to the stream. this.push(tap.buf.slice(0, pos)); } cb(); }; /** * Duplex stream to write object container files. * * @param schema * @param opts {Object} * * + `blockSize`, uncompressed. * + `codec` * + `codecs` * + `noCheck` * + `omitHeader`, useful to append to an existing block file. * */ function BlockEncoder(schema, opts) { opts = opts || {}; stream.Duplex.call(this, { allowHalfOpen: true, // To support async compressors. writableObjectMode: true }); var obj, type; if (types.Type.isType(schema)) { type = schema; schema = undefined; } else { // Keep full schema to be able to write it to the header later. obj = files.load(schema); type = types.createType(obj); schema = JSON.stringify(obj); } this._schema = schema; this._type = type; this._writeValue = function (tap, val) { try { this._type._write(tap, val); } catch (err) { this.emit('error', err); } }; this._blockSize = opts.blockSize || 65536; this._tap = new Tap(new Buffer(this._blockSize)); this._codecs = opts.codecs; this._codec = opts.codec || 'null'; this._compress = null; this._omitHeader = opts.omitHeader || false; this._blockCount = 0; this._syncMarker = opts.syncMarker || new utils.Lcg().nextBuffer(16); this._queue = new utils.OrderedQueue(); this._pending = 0; this._finished = false; this._needPush = false; this.on('finish', function () { this._finished = true; if (this._blockCount) { this._flushChunk(); } }); } util.inherits(BlockEncoder, stream.Duplex); BlockEncoder.getDefaultCodecs = function () { return { 'null': function (buf, cb) { cb(null, buf); }, 'deflate': zlib.deflateRaw }; }; BlockEncoder.prototype._write = function (val, encoding, cb) { var codec = this._codec; this._compress = (this._codecs || BlockEncoder.getDefaultCodecs())[codec]; if (!this._compress) { this.emit('error', new Error(f('unsupported codec: %s', codec))); return; } if (!this._omitHeader) { var meta = { 'avro.schema': new Buffer(this._schema || this._type.getSchema()), 'avro.codec': new Buffer(this._codec) }; var Header = HEADER_TYPE.getRecordConstructor(); var header = new Header(MAGIC_BYTES, meta, this._syncMarker); this.push(header.toBuffer()); } this._write = this._writeChunk; this._write(val, encoding, cb); }; BlockEncoder.prototype._writeChunk = function (val, encoding, cb) { var tap = this._tap; var pos = tap.pos; var flushing = false; this._writeValue(tap, val); if (!tap.isValid()) { if (pos) { this._flushChunk(pos, cb); flushing = true; } var len = tap.pos - pos; if (len > this._blockSize) { // Not enough space for last written object, need to resize. this._blockSize = len * 2; } tap.buf = new Buffer(this._blockSize); tap.pos = 0; this._writeValue(tap, val); // Rewrite last failed write. } this._blockCount++; if (!flushing) { cb(); } }; BlockEncoder.prototype._flushChunk = function (pos, cb) { var tap = this._tap; pos = pos || tap.pos; this._compress(tap.buf.slice(0, pos), this._createBlockCallback(cb)); this._blockCount = 0; }; BlockEncoder.prototype._read = function () { var self = this; var data = this._queue.pop(); if (!data) { if (this._finished && !this._pending) { process.nextTick(function () { self.push(null); }); } else { this._needPush = true; } return; } this.push(LONG_TYPE.toBuffer(data.count, true)); this.push(LONG_TYPE.toBuffer(data.buf.length, true)); this.push(data.buf); this.push(this._syncMarker); if (!this._finished) { data.cb(); } }; BlockEncoder.prototype._createBlockCallback = function (cb) { var self = this; var index = this._index++; var count = this._blockCount; this._pending++; return function (err, data) { if (err) { self.emit('error', err); return; } self._pending--; self._queue.push(new BlockData(index, data, cb, count)); if (self._needPush) { self._needPush = false; self._read(); } }; }; // Helpers. /** * An indexed block. * * This can be used to preserve block order since compression and decompression * can cause some some blocks to be returned out of order. The count is only * used when encoding. * */ function BlockData(index, buf, cb, count) { this.index = index; this.buf = buf; this.cb = cb; this.count = count | 0; } /** * Maybe get a block. * */ function tryReadBlock(tap) { var pos = tap.pos; var block = BLOCK_TYPE._read(tap); if (!tap.isValid()) { tap.pos = pos; return null; } return block; } /** * Create bytes consumer, either reading or skipping records. * */ function createReader(noDecode, type) { if (noDecode) { return (function (skipper) { return function (tap) { var pos = tap.pos; skipper(tap); return tap.buf.slice(pos, tap.pos); }; })(type._skip); } else { return function (tap) { return type._read(tap); }; } } /** * Copy a buffer. * * This avoids having to create a slice of the original buffer. * */ function copyBuffer(buf, pos, len) { var copy = new Buffer(len); buf.copy(copy, 0, pos, pos + len); return copy; } module.exports = { HEADER_TYPE: HEADER_TYPE, // For tests. MAGIC_BYTES: MAGIC_BYTES, // Idem. streams: { BlockDecoder: BlockDecoder, BlockEncoder: BlockEncoder, RawDecoder: RawDecoder, RawEncoder: RawEncoder } }; /***/ }, /* 3 */ /***/ function(module, exports, __webpack_require__) { /* jshint node: true */ 'use strict'; /** * Filesystem specifics. * * This module contains functions only used by node.js. It is shimmed by * another module when `avsc` is required from `browserify`. * */ var fs = __webpack_require__(4); /** * Try to load a schema. * * This method will attempt to load schemas from a file if the schema passed is * a string which isn't valid JSON and contains at least one slash. * */ function load(schema) { var obj; if (typeof schema == 'string' && schema !== 'null') { // This last predicate is to allow `avro.parse('null')` to work similarly // to `avro.parse('int')` and other primitives (null needs to be handled // separately since it is also a valid JSON identifier). try { obj = JSON.parse(schema); } catch (err) { if (~schema.indexOf('/')) { // This can't be a valid name, so we interpret is as a filepath. This // makes is always feasible to read a file, independent of its name // (i.e. even if its name is valid JSON), by prefixing it with `./`. obj = JSON.parse(fs.readFileSync(schema)); } } } if (obj === undefined) { obj = schema; } return obj; } /** * Default file loading function for assembling IDLs. * */ function createImportHook() { var imports = {}; return function (fpath, kind, cb) { if (imports[fpath]) { // Already imported, return nothing to avoid duplicating attributes. process.nextTick(cb); return; } imports[fpath] = true; fs.readFile(fpath, {encoding: 'utf8'}, cb); }; } module.exports = { createImportHook: createImportHook, load: load }; /***/ }, /* 4 */ /***/ function(module, exports) { /***/ }, /* 5 */ /***/ function(module, exports, __webpack_require__) { /* jshint node: true */ // TODO: Use `toFastProperties` on type reverse indices. // TODO: Allow configuring when to write the size when writing arrays and maps, // and customizing their block size. // TODO: Code-generate `compare` and `clone` record and union methods. 'use strict'; /** * This module defines all Avro data types and their serialization logic. * */ var utils = __webpack_require__(6), buffer = __webpack_require__(8), // For `SlowBuffer`. util = __webpack_require__(9); // Convenience imports. var Tap = utils.Tap; var f = util.format; // All non-union concrete (i.e. non-logical) Avro types. var TYPES = { 'array': ArrayType, 'boolean': BooleanType, 'bytes': BytesType, 'double': DoubleType, 'enum': EnumType, 'error': RecordType, 'fixed': FixedType, 'float': FloatType, 'int': IntType, 'long': LongType, 'map': MapType, 'null': NullType, 'record': RecordType, 'string': StringType }; // Valid (field, type, and symbol) name regex. var NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; // Random generator. var RANDOM = new utils.Lcg(); // Encoding tap (shared for performance). var TAP = new Tap(new buffer.SlowBuffer(1024)); // Currently active logical type, used for name redirection. var LOGICAL_TYPE = null; // Variable used to decide whether to include logical attributes when getting a // type's schema. A global variable is the simplest way to "pass an argument" // to JSON stringify's replacer function. var EXPORT_ATTRS = false; /** * Schema parsing entry point. * * It isn't exposed directly but called from `parse` inside `index.js` (node) * or `avsc.js` (browserify) which each add convenience functionality. * */ function createType(attrs, opts) { if (attrs === null) { // Let's be helpful for this common error. throw new Error('invalid type: null (did you mean "null"?)'); } if (Type.isType(attrs)) { return attrs; } opts = opts || {}; opts.registry = opts.registry || {}; var type; if (typeof attrs == 'string') { // Type reference. attrs = qualify(attrs, opts.namespace); type = opts.registry[attrs]; if (type) { // Type was already defined, return it. return type; } if (isPrimitive(attrs)) { // Reference to a primitive type. These are also defined names by default // so we create the appropriate type and it to the registry for future // reference. return opts.registry[attrs] = createType({type: attrs}, opts); } throw new Error(f('undefined type name: %s', attrs)); } if (opts.typeHook && (type = opts.typeHook(attrs, opts))) { if (!Type.isType(type)) { throw new Error(f('invalid typehook return value: %j', type)); } return type; } if (attrs.logicalType && opts.logicalTypes && !LOGICAL_TYPE) { var DerivedType = opts.logicalTypes[attrs.logicalType]; if (DerivedType) { var namespace = opts.namespace; var registry = {}; Object.keys(opts.registry).forEach(function (key) { registry[key] = opts.registry[key]; }); try { return new DerivedType(attrs, opts); } catch (err) { if (opts.assertLogicalTypes) { // The spec mandates that we fall through to the underlying type if // the logical type is invalid. We provide this option to ease // debugging. throw err; } LOGICAL_TYPE = null; opts.namespace = namespace; opts.registry = registry; } } } if (Array.isArray(attrs)) { // Union. var UnionType = opts.wrapUnions ? WrappedUnionType : UnwrappedUnionType; type = new UnionType(attrs, opts); } else { // New type definition. type = (function (typeName) { var Type = TYPES[typeName]; if (Type === undefined) { throw new Error(f('unknown type: %j', typeName)); } return new Type(attrs, opts); })(attrs.type); } return type; } /** * "Abstract" base Avro type. * * This class' constructor will register any named types to support recursive * schemas. All type values are represented in memory similarly to their JSON * representation, except for: * * + `bytes` and `fixed` which are represented as `Buffer`s. * + `union`s which will be "unwrapped" unless the `wrapUnions` option is set. * * See individual subclasses for details. * */ function Type(attrs, opts) { var type = LOGICAL_TYPE || this; LOGICAL_TYPE = null; // Lazily instantiated hash string. It will be generated the first time the // type's default fingerprint is computed (for example when using `equals`). this._hs = undefined; this._name = undefined; this._aliases = undefined; if (attrs) { // This is a complex (i.e. non-primitive) type. var name = attrs.name; var namespace = attrs.namespace === undefined ? opts && opts.namespace : attrs.namespace; if (name !== undefined) { // This isn't an anonymous type. name = qualify(name, namespace); if (isPrimitive(name)) { // Avro doesn't allow redefining primitive names. throw new Error(f('cannot rename primitive type: %j', name)); } var registry = opts && opts.registry; if (registry) { if (registry[name] !== undefined) { throw new Error(f('duplicate type name: %s', name)); } registry[name] = type; } } else if (opts && opts.noAnonymousTypes) { throw new Error(f('missing name property in schema: %j', attrs)); } this._name = name; this._aliases = attrs.aliases ? attrs.aliases.map(function (s) { return qualify(s, namespace); }) : []; } } Type.isType = function (/* any, [prefix] ... */) { var l = arguments.length; if (!l) { return false; } var any = arguments[0]; if ( !any || typeof any._update != 'function' || typeof any.getTypeName != 'function' ) { // Not fool-proof, but most likely good enough. return false; } if (l === 1) { // No type names specified, we are done. return true; } // We check if at least one of the prefixes matches. var typeName = any.getTypeName(); var i; for (i = 1; i < l; i++) { if (typeName.indexOf(arguments[i]) === 0) { return true; } } return false; }; Type.__reset = function (size) { TAP.buf = new buffer.SlowBuffer(size); }; Type.prototype.createResolver = function (type, opts) { if (!Type.isType(type)) { // More explicit error message than the "incompatible type" thrown // otherwise (especially because of the overridden `toJSON` method). throw new Error(f('not a type: %j', type)); } if (!Type.isType(this, 'logical') && Type.isType(type, 'logical')) { // Trying to read a logical type as a built-in: unwrap the logical type. return this.createResolver(type._underlyingType, opts); } opts = opts || {}; opts.registry = opts.registry || {}; var resolver, key; if ( Type.isType(this, 'record', 'error') && Type.isType(type, 'record', 'error') ) { // We allow conversions between records and errors. key = this._name + ':' + type._name; // ':' is illegal in Avro type names. resolver = opts.registry[key]; if (resolver) { return resolver; } } resolver = new Resolver(this); if (key) { // Register resolver early for recursive schemas. opts.registry[key] = resolver; } if (Type.isType(type, 'union')) { var resolvers = type._types.map(function (t) { return this.createResolver(t, opts); }, this); resolver._read = function (tap) { var index = tap.readLong(); var resolver = resolvers[index]; if (resolver === undefined) { throw new Error(f('invalid union index: %s', index)); } return resolvers[index]._read(tap); }; } else { this._update(resolver, type, opts); } if (!resolver._read) { throw new Error(f('cannot read %s as %s', type, this)); } return resolver; }; Type.prototype.decode = function (buf, pos, resolver) { var tap = new Tap(buf, pos); var val = readValue(this, tap, resolver); if (!tap.isValid()) { return {value: undefined, offset: -1}; } return {value: val, offset: tap.pos}; }; Type.prototype.encode = function (val, buf, pos) { var tap = new Tap(buf, pos); this._write(tap, val); if (!tap.isValid()) { // Don't throw as there is no way to predict this. We also return the // number of missing bytes to ease resizing. return buf.length - tap.pos; } return tap.pos; }; Type.prototype.fromBuffer = function (buf, resolver, noCheck) { var tap = new Tap(buf); var val = readValue(this, tap, resolver, noCheck); if (!tap.isValid()) { throw new Error('truncated buffer'); } if (!noCheck && tap.pos < buf.length) { throw new Error('trailing data'); } return val; }; Type.prototype.toBuffer = function (val) { TAP.pos = 0; this._write(TAP, val); if (!TAP.isValid()) { Type.__reset(2 * TAP.pos); TAP.pos = 0; this._write(TAP, val); } var buf = new Buffer(TAP.pos); TAP.buf.copy(buf, 0, 0, TAP.pos); return buf; }; Type.prototype.fromString = function (str) { return this._copy(JSON.parse(str), {coerce: 2}); }; Type.prototype.toString = function (val) { if (val === undefined) { // Consistent behavior with standard `toString` expectations. return this.getSchema({noDeref: true}); } return JSON.stringify(this._copy(val, {coerce: 3})); }; Type.prototype.clone = function (val, opts) { if (opts) { opts = { coerce: !!opts.coerceBuffers | 0, // Coerce JSON to Buffer. fieldHook: opts.fieldHook, qualifyNames: !!opts.qualifyNames, wrap: !!opts.wrapUnions | 0 // Wrap first match into union. }; return this._copy(val, opts); } else { // If no modifications are required, we can get by with a serialization // roundtrip (generally much faster than a standard deep copy). return this.fromBuffer(this.toBuffer(val)); } }; Type.prototype.isValid = function (val, opts) { // We only have a single flag for now, so no need to complicate things. var flags = (opts && opts.noUndeclaredFields) | 0; var errorHook = opts && opts.errorHook; var hook, path; if (errorHook) { path = []; hook = function (any, type) { errorHook.call(this, path.slice(), any, type, val); }; } return this._check(val, flags, hook, path); }; Type.prototype.compareBuffers = function (buf1, buf2) { return this._match(new Tap(buf1), new Tap(buf2)); }; Type.prototype.getName = function (asBranch) { var type = Type.isType(this, 'logical') ? this._underlyingType : this; if (type._name || !asBranch) { return type._name; } return Type.isType(this, 'union') ? undefined : type.getTypeName(); }; Type.prototype.getSchema = function (opts) { return stringify(this, opts); }; Type.prototype.equals = function (type) { return ( Type.isType(type) && this.getFingerprint().equals(type.getFingerprint()) ); }; Type.prototype.getFingerprint = function (algorithm) { if (!algorithm) { if (!this._hs) { this._hs = utils.getHash(this.getSchema()).toString('binary'); } return new Buffer(this._hs, 'binary'); } else { return utils.getHash(this.getSchema(), algorithm); } }; Type.prototype.inspect = function () { var typeName = this.getTypeName(); var className = getClassName(typeName); if (isPrimitive(typeName)) { // The class name is sufficient to identify the type. return f('<%s>', className); } else { // We add a little metadata for convenience. var obj = JSON.parse(this.getSchema({exportAttrs: true, noDeref: true})); if (typeof obj == 'object' && !Type.isType(this, 'logical')) { obj.type = undefined; // Would be redundant with constructor name. } return f('<%s %j>', className, obj); } }; Type.prototype._check = utils.abstractFunction; Type.prototype._copy = utils.abstractFunction; Type.prototype._match = utils.abstractFunction; Type.prototype._read = utils.abstractFunction; Type.prototype._skip = utils.abstractFunction; Type.prototype._update = utils.abstractFunction; Type.prototype._write = utils.abstractFunction; Type.prototype.compare = utils.abstractFunction; Type.prototype.getTypeName = utils.abstractFunction; Type.prototype.random = utils.abstractFunction; // Implementations. /** * Base primitive Avro type. * * Most of the primitive types share the same cloning and resolution * mechanisms, provided by this class. This class also lets us conveniently * check whether a type is a primitive using `instanceof`. * */ function PrimitiveType() { Type.call(this); } util.inherits(PrimitiveType, Type); PrimitiveType.prototype._update = function (resolver, type) { if (type.constructor === this.constructor) { resolver._read = this._read; } }; PrimitiveType.prototype._copy = function (val) { this._check(val, undefined, throwInvalidError); return val; }; PrimitiveType.prototype.compare = utils.compare; PrimitiveType.prototype.toJSON = function () { return this.getTypeName(); }; /** * Nulls. * */ function NullType() { PrimitiveType.call(this); } util.inherits(NullType, PrimitiveType); NullType.prototype._check = function (val, flags, hook) { var b = val === null; if (!b && hook) { hook(val, this); } return b; }; NullType.prototype._read = function () { return null; }; NullType.prototype._skip = function () {}; NullType.prototype._write = function (tap, val) { if (val !== null) { throwInvalidError(val, this); } }; NullType.prototype._match = function () { return 0; }; NullType.prototype.compare = NullType.prototype._match; NullType.prototype.getTypeName = function () { return 'null'; }; NullType.prototype.random = NullType.prototype._read; /** * Booleans. * */ function BooleanType() { PrimitiveType.call(this); } util.inherits(BooleanType, PrimitiveType); BooleanType.prototype._check = function (val, flags, hook) { var b = typeof val == 'boolean'; if (!b && hook) { hook(val, this); } return b; }; BooleanType.prototype._read = function (tap) { return tap.readBoolean(); }; BooleanType.prototype._skip = function (tap) { tap.skipBoolean(); }; BooleanType.prototype._write = function (tap, val) { if (typeof val != 'boolean') { throwInvalidError(val, this); } tap.writeBoolean(val); }; BooleanType.prototype._match = function (tap1, tap2) { return tap1.matchBoolean(tap2); }; BooleanType.prototype.getTypeName = function () { return 'boolean'; }; BooleanType.prototype.random = function () { return RANDOM.nextBoolean(); }; /** * Integers. * */ function IntType() { PrimitiveType.call(this); } util.inherits(IntType, PrimitiveType); IntType.prototype._check = function (val, flags, hook) { var b = val === (val | 0); if (!b && hook) { hook(val, this); } return b; }; IntType.prototype._read = function (tap) { return tap.readInt(); }; IntType.prototype._skip = function (tap) { tap.skipInt(); }; IntType.prototype._write = function (tap, val) { if (val !== (val | 0)) { throwInvalidError(val, this); } tap.writeInt(val); }; IntType.prototype._match = function (tap1, tap2) { return tap1.matchInt(tap2); }; IntType.prototype.getTypeName = function () { return 'int'; }; IntType.prototype.random = function () { return RANDOM.nextInt(1000) | 0; }; /** * Longs. * * We can't capture all the range unfortunately since JavaScript represents all * numbers internally as `double`s, so the default implementation plays safe * and throws rather than potentially silently change the data. See `__with` or * `AbstractLongType` below for a way to implement a custom long type. * */ function LongType() { PrimitiveType.call(this); } util.inherits(LongType, PrimitiveType); LongType.prototype._check = function (val, flags, hook) { var b = typeof val == 'number' && val % 1 === 0 && isSafeLong(val); if (!b && hook) { hook(val, this); } return b; }; LongType.prototype._read = function (tap) { var n = tap.readLong(); if (!isSafeLong(n)) { throw new Error('potential precision loss'); } return n; }; LongType.prototype._skip = function (tap) { tap.skipLong(); }; LongType.prototype._write = function (tap, val) { if (typeof val != 'number' || val % 1 || !isSafeLong(val)) { throwInvalidError(val, this); } tap.writeLong(val); }; LongType.prototype._match = function (tap1, tap2) { return tap1.matchLong(tap2); }; LongType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'int': case 'long': resolver._read = type._read; } }; LongType.prototype.getTypeName = function () { return 'long'; }; LongType.prototype.random = function () { return RANDOM.nextInt(); }; LongType.__with = function (methods, noUnpack) { methods = methods || {}; // Will give a more helpful error message. // We map some of the methods to a different name to be able to intercept // their input and output (otherwise we wouldn't be able to perform any // unpacking logic, and the type wouldn't work when nested). var mapping = { toBuffer: '_toBuffer', fromBuffer: '_fromBuffer', fromJSON: '_fromJSON', toJSON: '_toJSON', isValid: '_isValid', compare: 'compare' }; var type = new AbstractLongType(noUnpack); Object.keys(mapping).forEach(function (name) { if (methods[name] === undefined) { throw new Error(f('missing method implementation: %s', name)); } type[mapping[name]] = methods[name]; }); return type; }; /** * Floats. * */ function FloatType() { PrimitiveType.call(this); } util.inherits(FloatType, PrimitiveType); FloatType.prototype._check = function (val, flags, hook) { var b = typeof val == 'number'; if (!b && hook) { hook(val, this); } return b; }; FloatType.prototype._read = function (tap) { return tap.readFloat(); }; FloatType.prototype._skip = function (tap) { tap.skipFloat(); }; FloatType.prototype._write = function (tap, val) { if (typeof val != 'number') { throwInvalidError(val, this); } tap.writeFloat(val); }; FloatType.prototype._match = function (tap1, tap2) { return tap1.matchFloat(tap2); }; FloatType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'float': case 'int': case 'long': resolver._read = type._read; } }; FloatType.prototype.getTypeName = function () { return 'float'; }; FloatType.prototype.random = function () { return RANDOM.nextFloat(1e3); }; /** * Doubles. * */ function DoubleType() { PrimitiveType.call(this); } util.inherits(DoubleType, PrimitiveType); DoubleType.prototype._check = function (val, flags, hook) { var b = typeof val == 'number'; if (!b && hook) { hook(val, this); } return b; }; DoubleType.prototype._read = function (tap) { return tap.readDouble(); }; DoubleType.prototype._skip = function (tap) { tap.skipDouble(); }; DoubleType.prototype._write = function (tap, val) { if (typeof val != 'number') { throwInvalidError(val, this); } tap.writeDouble(val); }; DoubleType.prototype._match = function (tap1, tap2) { return tap1.matchDouble(tap2); }; DoubleType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'double': case 'float': case 'int': case 'long': resolver._read = type._read; } }; DoubleType.prototype.getTypeName = function () { return 'double'; }; DoubleType.prototype.random = function () { return RANDOM.nextFloat(); }; /** * Strings. * */ function StringType() { PrimitiveType.call(this); } util.inherits(StringType, PrimitiveType); StringType.prototype._check = function (val, flags, hook) { var b = typeof val == 'string'; if (!b && hook) { hook(val, this); } return b; }; StringType.prototype._read = function (tap) { return tap.readString(); }; StringType.prototype._skip = function (tap) { tap.skipString(); }; StringType.prototype._write = function (tap, val) { if (typeof val != 'string') { throwInvalidError(val, this); } tap.writeString(val); }; StringType.prototype._match = function (tap1, tap2) { return tap1.matchString(tap2); }; StringType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'bytes': case 'string': resolver._read = this._read; } }; StringType.prototype.getTypeName = function () { return 'string'; }; StringType.prototype.random = function () { return RANDOM.nextString(RANDOM.nextInt(32)); }; /** * Bytes. * * These are represented in memory as `Buffer`s rather than binary-encoded * strings. This is more efficient (when decoding/encoding from bytes, the * common use-case), idiomatic, and convenient. * * Note the coercion in `_copy`. * */ function BytesType() { PrimitiveType.call(this); } util.inherits(BytesType, PrimitiveType); BytesType.prototype._check = function (val, flags, hook) { var b = Buffer.isBuffer(val); if (!b && hook) { hook(val, this); } return b; }; BytesType.prototype._read = function (tap) { return tap.readBytes(); }; BytesType.prototype._skip = function (tap) { tap.skipBytes(); }; BytesType.prototype._write = function (tap, val) { if (!Buffer.isBuffer(val)) { throwInvalidError(val, this); } tap.writeBytes(val); }; BytesType.prototype._match = function (tap1, tap2) { return tap1.matchBytes(tap2); }; BytesType.prototype._update = StringType.prototype._update; BytesType.prototype._copy = function (obj, opts) { var buf; switch ((opts && opts.coerce) | 0) { case 3: // Coerce buffers to strings. this._check(obj, undefined, throwInvalidError); return obj.toString('binary'); case 2: // Coerce strings to buffers. if (typeof obj != 'string') { throw new Error(f('cannot coerce to buffer: %j', obj)); } buf = new Buffer(obj, 'binary'); this._check(buf, undefined, throwInvalidError); return buf; case 1: // Coerce buffer JSON representation to buffers. if (!isJsonBuffer(obj)) { throw new Error(f('cannot coerce to buffer: %j', obj)); } buf = new Buffer(obj.data); this._check(buf, undefined, throwInvalidError); return buf; default: // Copy buffer. this._check(obj, undefined, throwInvalidError); return new Buffer(obj); } }; BytesType.prototype.compare = Buffer.compare; BytesType.prototype.getTypeName = function () { return 'bytes'; }; BytesType.prototype.random = function () { return RANDOM.nextBuffer(RANDOM.nextInt(32)); }; /** * Base "abstract" Avro union type. * */ function UnionType(attrs, opts) { Type.call(this); if (!Array.isArray(attrs)) { throw new Error(f('non-array union schema: %j', attrs)); } if (!attrs.length) { throw new Error('empty union'); } this._types = attrs.map(function (obj) { return createType(obj, opts); }); this._branchIndices = {}; this._types.forEach(function (type, i) { if (Type.isType(type, 'union')) { throw new Error('unions cannot be directly nested'); } var branch = type.getName(true); if (this._branchIndices[branch] !== undefined) { throw new Error(f('duplicate union branch name: %j', branch)); } this._branchIndices[branch] = i; }, this); } util.inherits(UnionType, Type); UnionType.prototype._skip = function (tap) { this._types[tap.readLong()]._skip(tap); }; UnionType.prototype._match = function (tap1, tap2) { var n1 = tap1.readLong(); var n2 = tap2.readLong(); if (n1 === n2) { return this._types[n1]._match(tap1, tap2); } else { return n1 < n2 ? -1 : 1; } }; UnionType.prototype.getTypes = function () { return this._types.slice(); }; UnionType.prototype.toJSON = function () { return this._types; }; /** * "Natural" union type. * * This representation doesn't require a wrapping object and is therefore * simpler and generally closer to what users expect. However it cannot be used * to represent all Avro unions since some lead to ambiguities (e.g. if two * number types are in the union). * * Currently, this union supports at most one type in each of the categories * below: * * + `null` * + `boolean` * + `int`, `long`, `float`, `double` * + `string`, `enum` * + `bytes`, `fixed` * + `array` * + `map`, `record` * */ function UnwrappedUnionType(attrs, opts) { UnionType.call(this, attrs, opts); this._logicalBranches = null; this._bucketIndices = {}; this._types.forEach(function (type, index) { if (Type.isType(type, 'logical')) { if (!this._logicalBranches) { this._logicalBranches = []; } this._logicalBranches.push({index: index, type: type}); } else { var bucket = getTypeBucket(type); if (this._bucketIndices[bucket] !== undefined) { throw new Error(f('ambiguous unwrapped union: %j', this)); } this._bucketIndices[bucket] = index; } }, this); } util.inherits(UnwrappedUnionType, UnionType); UnwrappedUnionType.prototype._getIndex = function (val) { var index = this._bucketIndices[getValueBucket(val)]; if (this._logicalBranches) { // Slower path, we must run the value through all logical types. index = this._getLogicalIndex(val, index); } return index; }; UnwrappedUnionType.prototype._getLogicalIndex = function (any, index) { var logicalBranches = this._logicalBranches; var i, l, branch; for (i = 0, l = logicalBranches.length; i < l; i++) { branch = logicalBranches[i]; if (branch.type._check(any)) { if (index === undefined) { index = branch.index; } else { // More than one branch matches the value so we aren't guaranteed to // infer the correct type. We throw rather than corrupt data. This can // be fixed by "tightening" the logical types. throw new Error('ambiguous conversion'); } } } return index; }; UnwrappedUnionType.prototype._check = function (val, flags, hook, path) { var index = this._getIndex(val); var b = index !== undefined; if (b) { return this._types[index]._check(val, flags, hook, path); } if (hook) { hook(val, this); } return b; }; UnwrappedUnionType.prototype._read = function (tap) { var index = tap.readLong(); var branchType = this._types[index]; if (branchType) { return branchType._read(tap); } else { throw new Error(f('invalid union index: %s', index)); } }; UnwrappedUnionType.prototype._write = function (tap, val) { var index = this._getIndex(val); if (index === undefined) { throwInvalidError(val, this); } tap.writeLong(index); if (val !== null) { this._types[index]._write(tap, val); } }; UnwrappedUnionType.prototype._update = function (resolver, type, opts) { // jshint -W083 // (The loop exits after the first function is created.) var i, l, typeResolver; for (i = 0, l = this._types.length; i < l; i++) { try { typeResolver = this._types[i].createResolver(type, opts); } catch (err) { continue; } resolver._read = function (tap) { return typeResolver._read(tap); }; return; } }; UnwrappedUnionType.prototype._copy = function (val, opts) { var coerce = opts && opts.coerce | 0; var wrap = opts && opts.wrap | 0; var index; if (wrap === 2) { // We are parsing a default, so alw