UNPKG

cdap-avsc

Version:

This project is a clone of mtth/avsc repo with modifications required by CDAP

1,875 lines (1,644 loc) 75.8 kB
/* jshint node: true */ // TODO: Use `toFastProperties` on type reverse indices. // TODO: Allow configuring when to write the size when writing arrays and maps, // and customizing their block size. // TODO: Code-generate `compare` and `clone` record and union methods. 'use strict'; /** * This module defines all Avro data types and their serialization logic. * */ var utils = require('./utils'), buffer = require('buffer'), // For `SlowBuffer`. util = require('util'); // Convenience imports. var Tap = utils.Tap; var f = util.format; // All non-union concrete (i.e. non-logical) Avro types. var TYPES = { 'array': ArrayType, 'boolean': BooleanType, 'bytes': BytesType, 'double': DoubleType, 'enum': EnumType, 'error': RecordType, 'fixed': FixedType, 'float': FloatType, 'int': IntType, 'long': LongType, 'map': MapType, 'null': NullType, 'record': RecordType, 'string': StringType }; // Addition by Ajai // Allow hyphens to be part of field name (including enums) // Valid (field, type, and symbol) name regex. var NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_-]*$/; // Random generator. var RANDOM = new utils.Lcg(); // Encoding tap (shared for performance). var TAP = new Tap(new buffer.SlowBuffer(1024)); // Currently active logical type, used for name redirection. var LOGICAL_TYPE = null; // Variable used to decide whether to include logical attributes when getting a // type's schema. A global variable is the simplest way to "pass an argument" // to JSON stringify's replacer function. var EXPORT_ATTRS = false; /** * Schema parsing entry point. * * It isn't exposed directly but called from `parse` inside `index.js` (node) * or `avsc.js` (browserify) which each add convenience functionality. * */ function createType(attrs, opts) { if (attrs === null) { // Let's be helpful for this common error. throw new Error('invalid type: null (did you mean "null"?)'); } if (Type.isType(attrs)) { return attrs; } opts = opts || {}; opts.registry = opts.registry || {}; var type; if (opts.typeHook && (type = opts.typeHook(attrs, opts))) { if (!Type.isType(type)) { throw new Error(f('invalid typehook return value: %j', type)); } return type; } if (typeof attrs == 'string') { // Type reference. attrs = qualify(attrs, opts.namespace); type = opts.registry[attrs]; if (type) { // Type was already defined, return it. return type; } if (isPrimitive(attrs)) { // Reference to a primitive type. These are also defined names by default // so we create the appropriate type and it to the registry for future // reference. return opts.registry[attrs] = createType({type: attrs}, opts); } throw new Error(f('undefined type name: %s', attrs)); } if (attrs.logicalType && opts.logicalTypes && !LOGICAL_TYPE) { var DerivedType = opts.logicalTypes[attrs.logicalType]; if (DerivedType) { var namespace = opts.namespace; var registry = {}; Object.keys(opts.registry).forEach(function (key) { registry[key] = opts.registry[key]; }); try { return new DerivedType(attrs, opts); } catch (err) { if (opts.assertLogicalTypes) { // The spec mandates that we fall through to the underlying type if // the logical type is invalid. We provide this option to ease // debugging. throw err; } LOGICAL_TYPE = null; opts.namespace = namespace; opts.registry = registry; } } } if (Array.isArray(attrs)) { // Union. var UnionType = opts.wrapUnions ? WrappedUnionType : UnwrappedUnionType; type = new UnionType(attrs, opts); } else { // New type definition. type = (function (typeName) { var Type = TYPES[typeName]; if (Type === undefined) { throw new Error(f('unknown type: %j', typeName)); } return new Type(attrs, opts); })(attrs.type); } return type; } /** * "Abstract" base Avro type. * * This class' constructor will register any named types to support recursive * schemas. All type values are represented in memory similarly to their JSON * representation, except for: * * + `bytes` and `fixed` which are represented as `Buffer`s. * + `union`s which will be "unwrapped" unless the `wrapUnions` option is set. * * See individual subclasses for details. * */ function Type(attrs, opts) { var type = LOGICAL_TYPE || this; LOGICAL_TYPE = null; // Lazily instantiated hash string. It will be generated the first time the // type's default fingerprint is computed (for example when using `equals`). this._hs = undefined; this._name = undefined; this._aliases = undefined; if (attrs) { // This is a complex (i.e. non-primitive) type. var name = attrs.name; var namespace = attrs.namespace === undefined ? opts && opts.namespace : attrs.namespace; if (name !== undefined) { // This isn't an anonymous type. name = qualify(name, namespace); if (isPrimitive(name)) { // Avro doesn't allow redefining primitive names. throw new Error(f('cannot rename primitive type: %j', name)); } var registry = opts && opts.registry; if (registry) { if (registry[name] !== undefined) { throw new Error(f('duplicate type name: %s', name)); } registry[name] = type; } } else if (opts && opts.noAnonymousTypes) { throw new Error(f('missing name property in schema: %j', attrs)); } this._name = name; this._aliases = attrs.aliases ? attrs.aliases.map(function (s) { return qualify(s, namespace); }) : []; } } Type.isType = function (/* any, [prefix] ... */) { var l = arguments.length; if (!l) { return false; } var any = arguments[0]; if ( !any || typeof any._update != 'function' || typeof any.getTypeName != 'function' ) { // Not fool-proof, but most likely good enough. return false; } if (l === 1) { // No type names specified, we are done. return true; } // We check if at least one of the prefixes matches. var typeName = any.getTypeName(); var i; for (i = 1; i < l; i++) { if (typeName.indexOf(arguments[i]) === 0) { return true; } } return false; }; Type.__reset = function (size) { TAP.buf = new buffer.SlowBuffer(size); }; Type.prototype.createResolver = function (type, opts) { if (!Type.isType(type)) { // More explicit error message than the "incompatible type" thrown // otherwise (especially because of the overridden `toJSON` method). throw new Error(f('not a type: %j', type)); } if (!Type.isType(this, 'logical') && Type.isType(type, 'logical')) { // Trying to read a logical type as a built-in: unwrap the logical type. return this.createResolver(type._underlyingType, opts); } opts = opts || {}; opts.registry = opts.registry || {}; var resolver, key; if ( Type.isType(this, 'record', 'error') && Type.isType(type, 'record', 'error') ) { // We allow conversions between records and errors. key = this._name + ':' + type._name; // ':' is illegal in Avro type names. resolver = opts.registry[key]; if (resolver) { return resolver; } } resolver = new Resolver(this); if (key) { // Register resolver early for recursive schemas. opts.registry[key] = resolver; } if (Type.isType(type, 'union')) { var resolvers = type._types.map(function (t) { return this.createResolver(t, opts); }, this); resolver._read = function (tap) { var index = tap.readLong(); var resolver = resolvers[index]; if (resolver === undefined) { throw new Error(f('invalid union index: %s', index)); } return resolvers[index]._read(tap); }; } else { this._update(resolver, type, opts); } if (!resolver._read) { throw new Error(f('cannot read %s as %s', type, this)); } return resolver; }; Type.prototype.decode = function (buf, pos, resolver) { var tap = new Tap(buf, pos); var val = readValue(this, tap, resolver); if (!tap.isValid()) { return {value: undefined, offset: -1}; } return {value: val, offset: tap.pos}; }; Type.prototype.encode = function (val, buf, pos) { var tap = new Tap(buf, pos); this._write(tap, val); if (!tap.isValid()) { // Don't throw as there is no way to predict this. We also return the // number of missing bytes to ease resizing. return buf.length - tap.pos; } return tap.pos; }; Type.prototype.fromBuffer = function (buf, resolver, noCheck) { var tap = new Tap(buf); var val = readValue(this, tap, resolver, noCheck); if (!tap.isValid()) { throw new Error('truncated buffer'); } if (!noCheck && tap.pos < buf.length) { throw new Error('trailing data'); } return val; }; Type.prototype.toBuffer = function (val) { TAP.pos = 0; this._write(TAP, val); if (!TAP.isValid()) { Type.__reset(2 * TAP.pos); TAP.pos = 0; this._write(TAP, val); } var buf = new Buffer(TAP.pos); TAP.buf.copy(buf, 0, 0, TAP.pos); return buf; }; Type.prototype.fromString = function (str) { return this._copy(JSON.parse(str), {coerce: 2}); }; Type.prototype.toString = function (val) { if (val === undefined) { // Consistent behavior with standard `toString` expectations. return this.getSchema({noDeref: true}); } return JSON.stringify(this._copy(val, {coerce: 3})); }; Type.prototype.clone = function (val, opts) { if (opts) { opts = { coerce: !!opts.coerceBuffers | 0, // Coerce JSON to Buffer. fieldHook: opts.fieldHook, qualifyNames: !!opts.qualifyNames, wrap: !!opts.wrapUnions | 0 // Wrap first match into union. }; return this._copy(val, opts); } else { // If no modifications are required, we can get by with a serialization // roundtrip (generally much faster than a standard deep copy). return this.fromBuffer(this.toBuffer(val)); } }; Type.prototype.isValid = function (val, opts) { // We only have a single flag for now, so no need to complicate things. var flags = (opts && opts.noUndeclaredFields) | 0; var errorHook = opts && opts.errorHook; var hook, path; if (errorHook) { path = []; hook = function (any, type) { errorHook.call(this, path.slice(), any, type, val); }; } return this._check(val, flags, hook, path); }; Type.prototype.compareBuffers = function (buf1, buf2) { return this._match(new Tap(buf1), new Tap(buf2)); }; Type.prototype.getName = function (asBranch) { var type = Type.isType(this, 'logical') ? this._underlyingType : this; if (type._name || !asBranch) { return type._name; } return Type.isType(this, 'union') ? undefined : type.getTypeName(); }; Type.prototype.getSchema = function (opts) { return stringify(this, opts); }; Type.prototype.equals = function (type) { return ( Type.isType(type) && this.getFingerprint().equals(type.getFingerprint()) ); }; Type.prototype.getFingerprint = function (algorithm) { if (!algorithm) { if (!this._hs) { this._hs = utils.getHash(this.getSchema()).toString('binary'); } return new Buffer(this._hs, 'binary'); } else { return utils.getHash(this.getSchema(), algorithm); } }; Type.prototype.inspect = function () { var typeName = this.getTypeName(); var className = getClassName(typeName); if (isPrimitive(typeName)) { // The class name is sufficient to identify the type. return f('<%s>', className); } else { // We add a little metadata for convenience. var obj = JSON.parse(this.getSchema({exportAttrs: true, noDeref: true})); if (typeof obj == 'object' && !Type.isType(this, 'logical')) { obj.type = undefined; // Would be redundant with constructor name. } return f('<%s %j>', className, obj); } }; Type.prototype._check = utils.abstractFunction; Type.prototype._copy = utils.abstractFunction; Type.prototype._match = utils.abstractFunction; Type.prototype._read = utils.abstractFunction; Type.prototype._skip = utils.abstractFunction; Type.prototype._update = utils.abstractFunction; Type.prototype._write = utils.abstractFunction; Type.prototype.compare = utils.abstractFunction; Type.prototype.getTypeName = utils.abstractFunction; Type.prototype.random = utils.abstractFunction; // Implementations. /** * Base primitive Avro type. * * Most of the primitive types share the same cloning and resolution * mechanisms, provided by this class. This class also lets us conveniently * check whether a type is a primitive using `instanceof`. * */ function PrimitiveType() { Type.call(this); } util.inherits(PrimitiveType, Type); PrimitiveType.prototype._update = function (resolver, type) { if (type.constructor === this.constructor) { resolver._read = this._read; } }; PrimitiveType.prototype._copy = function (val) { this._check(val, undefined, throwInvalidError); return val; }; PrimitiveType.prototype.compare = utils.compare; PrimitiveType.prototype.toJSON = function () { return this.getTypeName(); }; /** * Nulls. * */ function NullType() { PrimitiveType.call(this); } util.inherits(NullType, PrimitiveType); NullType.prototype._check = function (val, flags, hook) { var b = val === null; if (!b && hook) { hook(val, this); } return b; }; NullType.prototype._read = function () { return null; }; NullType.prototype._skip = function () {}; NullType.prototype._write = function (tap, val) { if (val !== null) { throwInvalidError(val, this); } }; NullType.prototype._match = function () { return 0; }; NullType.prototype.compare = NullType.prototype._match; NullType.prototype.getTypeName = function () { return 'null'; }; NullType.prototype.random = NullType.prototype._read; /** * Booleans. * */ function BooleanType() { PrimitiveType.call(this); } util.inherits(BooleanType, PrimitiveType); BooleanType.prototype._check = function (val, flags, hook) { var b = typeof val == 'boolean'; if (!b && hook) { hook(val, this); } return b; }; BooleanType.prototype._read = function (tap) { return tap.readBoolean(); }; BooleanType.prototype._skip = function (tap) { tap.skipBoolean(); }; BooleanType.prototype._write = function (tap, val) { if (typeof val != 'boolean') { throwInvalidError(val, this); } tap.writeBoolean(val); }; BooleanType.prototype._match = function (tap1, tap2) { return tap1.matchBoolean(tap2); }; BooleanType.prototype.getTypeName = function () { return 'boolean'; }; BooleanType.prototype.random = function () { return RANDOM.nextBoolean(); }; /** * Integers. * */ function IntType() { PrimitiveType.call(this); } util.inherits(IntType, PrimitiveType); IntType.prototype._check = function (val, flags, hook) { var b = val === (val | 0); if (!b && hook) { hook(val, this); } return b; }; IntType.prototype._read = function (tap) { return tap.readInt(); }; IntType.prototype._skip = function (tap) { tap.skipInt(); }; IntType.prototype._write = function (tap, val) { if (val !== (val | 0)) { throwInvalidError(val, this); } tap.writeInt(val); }; IntType.prototype._match = function (tap1, tap2) { return tap1.matchInt(tap2); }; IntType.prototype.getTypeName = function () { return 'int'; }; IntType.prototype.random = function () { return RANDOM.nextInt(1000) | 0; }; /** * Longs. * * We can't capture all the range unfortunately since JavaScript represents all * numbers internally as `double`s, so the default implementation plays safe * and throws rather than potentially silently change the data. See `__with` or * `AbstractLongType` below for a way to implement a custom long type. * */ function LongType() { PrimitiveType.call(this); } util.inherits(LongType, PrimitiveType); LongType.prototype._check = function (val, flags, hook) { var b = typeof val == 'number' && val % 1 === 0 && isSafeLong(val); if (!b && hook) { hook(val, this); } return b; }; LongType.prototype._read = function (tap) { var n = tap.readLong(); if (!isSafeLong(n)) { throw new Error('potential precision loss'); } return n; }; LongType.prototype._skip = function (tap) { tap.skipLong(); }; LongType.prototype._write = function (tap, val) { if (typeof val != 'number' || val % 1 || !isSafeLong(val)) { throwInvalidError(val, this); } tap.writeLong(val); }; LongType.prototype._match = function (tap1, tap2) { return tap1.matchLong(tap2); }; LongType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'int': case 'long': resolver._read = type._read; } }; LongType.prototype.getTypeName = function () { return 'long'; }; LongType.prototype.random = function () { return RANDOM.nextInt(); }; LongType.__with = function (methods, noUnpack) { methods = methods || {}; // Will give a more helpful error message. // We map some of the methods to a different name to be able to intercept // their input and output (otherwise we wouldn't be able to perform any // unpacking logic, and the type wouldn't work when nested). var mapping = { toBuffer: '_toBuffer', fromBuffer: '_fromBuffer', fromJSON: '_fromJSON', toJSON: '_toJSON', isValid: '_isValid', compare: 'compare' }; var type = new AbstractLongType(noUnpack); Object.keys(mapping).forEach(function (name) { if (methods[name] === undefined) { throw new Error(f('missing method implementation: %s', name)); } type[mapping[name]] = methods[name]; }); return type; }; /** * Floats. * */ function FloatType() { PrimitiveType.call(this); } util.inherits(FloatType, PrimitiveType); FloatType.prototype._check = function (val, flags, hook) { var b = typeof val == 'number'; if (!b && hook) { hook(val, this); } return b; }; FloatType.prototype._read = function (tap) { return tap.readFloat(); }; FloatType.prototype._skip = function (tap) { tap.skipFloat(); }; FloatType.prototype._write = function (tap, val) { if (typeof val != 'number') { throwInvalidError(val, this); } tap.writeFloat(val); }; FloatType.prototype._match = function (tap1, tap2) { return tap1.matchFloat(tap2); }; FloatType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'float': case 'int': case 'long': resolver._read = type._read; } }; FloatType.prototype.getTypeName = function () { return 'float'; }; FloatType.prototype.random = function () { return RANDOM.nextFloat(1e3); }; /** * Doubles. * */ function DoubleType() { PrimitiveType.call(this); } util.inherits(DoubleType, PrimitiveType); DoubleType.prototype._check = function (val, flags, hook) { var b = typeof val == 'number'; if (!b && hook) { hook(val, this); } return b; }; DoubleType.prototype._read = function (tap) { return tap.readDouble(); }; DoubleType.prototype._skip = function (tap) { tap.skipDouble(); }; DoubleType.prototype._write = function (tap, val) { if (typeof val != 'number') { throwInvalidError(val, this); } tap.writeDouble(val); }; DoubleType.prototype._match = function (tap1, tap2) { return tap1.matchDouble(tap2); }; DoubleType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'double': case 'float': case 'int': case 'long': resolver._read = type._read; } }; DoubleType.prototype.getTypeName = function () { return 'double'; }; DoubleType.prototype.random = function () { return RANDOM.nextFloat(); }; /** * Strings. * */ function StringType() { PrimitiveType.call(this); } util.inherits(StringType, PrimitiveType); StringType.prototype._check = function (val, flags, hook) { var b = typeof val == 'string'; if (!b && hook) { hook(val, this); } return b; }; StringType.prototype._read = function (tap) { return tap.readString(); }; StringType.prototype._skip = function (tap) { tap.skipString(); }; StringType.prototype._write = function (tap, val) { if (typeof val != 'string') { throwInvalidError(val, this); } tap.writeString(val); }; StringType.prototype._match = function (tap1, tap2) { return tap1.matchString(tap2); }; StringType.prototype._update = function (resolver, type) { switch (type.getTypeName()) { case 'bytes': case 'string': resolver._read = this._read; } }; StringType.prototype.getTypeName = function () { return 'string'; }; StringType.prototype.random = function () { return RANDOM.nextString(RANDOM.nextInt(32)); }; /** * Bytes. * * These are represented in memory as `Buffer`s rather than binary-encoded * strings. This is more efficient (when decoding/encoding from bytes, the * common use-case), idiomatic, and convenient. * * Note the coercion in `_copy`. * */ function BytesType() { PrimitiveType.call(this); } util.inherits(BytesType, PrimitiveType); BytesType.prototype._check = function (val, flags, hook) { var b = Buffer.isBuffer(val); if (!b && hook) { hook(val, this); } return b; }; BytesType.prototype._read = function (tap) { return tap.readBytes(); }; BytesType.prototype._skip = function (tap) { tap.skipBytes(); }; BytesType.prototype._write = function (tap, val) { if (!Buffer.isBuffer(val)) { throwInvalidError(val, this); } tap.writeBytes(val); }; BytesType.prototype._match = function (tap1, tap2) { return tap1.matchBytes(tap2); }; BytesType.prototype._update = StringType.prototype._update; BytesType.prototype._copy = function (obj, opts) { var buf; switch ((opts && opts.coerce) | 0) { case 3: // Coerce buffers to strings. this._check(obj, undefined, throwInvalidError); return obj.toString('binary'); case 2: // Coerce strings to buffers. if (typeof obj != 'string') { throw new Error(f('cannot coerce to buffer: %j', obj)); } buf = new Buffer(obj, 'binary'); this._check(buf, undefined, throwInvalidError); return buf; case 1: // Coerce buffer JSON representation to buffers. if (!isJsonBuffer(obj)) { throw new Error(f('cannot coerce to buffer: %j', obj)); } buf = new Buffer(obj.data); this._check(buf, undefined, throwInvalidError); return buf; default: // Copy buffer. this._check(obj, undefined, throwInvalidError); return new Buffer(obj); } }; BytesType.prototype.compare = Buffer.compare; BytesType.prototype.getTypeName = function () { return 'bytes'; }; BytesType.prototype.random = function () { return RANDOM.nextBuffer(RANDOM.nextInt(32)); }; /** * Base "abstract" Avro union type. * */ function UnionType(attrs, opts) { Type.call(this); if (!Array.isArray(attrs)) { throw new Error(f('non-array union schema: %j', attrs)); } if (!attrs.length) { throw new Error('empty union'); } this._types = attrs.map(function (obj) { return createType(obj, opts); }); this._branchIndices = {}; this._types.forEach(function (type, i) { if (Type.isType(type, 'union')) { throw new Error('unions cannot be directly nested'); } var branch = type.getName(true); if (this._branchIndices[branch] !== undefined) { throw new Error(f('duplicate union branch name: %j', branch)); } this._branchIndices[branch] = i; }, this); } util.inherits(UnionType, Type); UnionType.prototype._skip = function (tap) { this._types[tap.readLong()]._skip(tap); }; UnionType.prototype._match = function (tap1, tap2) { var n1 = tap1.readLong(); var n2 = tap2.readLong(); if (n1 === n2) { return this._types[n1]._match(tap1, tap2); } else { return n1 < n2 ? -1 : 1; } }; UnionType.prototype.getTypes = function () { return this._types.slice(); }; UnionType.prototype.toJSON = function () { return this._types; }; /** * "Natural" union type. * * This representation doesn't require a wrapping object and is therefore * simpler and generally closer to what users expect. However it cannot be used * to represent all Avro unions since some lead to ambiguities (e.g. if two * number types are in the union). * * Currently, this union supports at most one type in each of the categories * below: * * + `null` * + `boolean` * + `int`, `long`, `float`, `double` * + `string`, `enum` * + `bytes`, `fixed` * + `array` * + `map`, `record` * */ function UnwrappedUnionType(attrs, opts) { UnionType.call(this, attrs, opts); this._logicalBranches = null; this._bucketIndices = {}; this._types.forEach(function (type, index) { if (Type.isType(type, 'logical')) { if (!this._logicalBranches) { this._logicalBranches = []; } this._logicalBranches.push({index: index, type: type}); } else { var bucket = getTypeBucket(type); if (this._bucketIndices[bucket] !== undefined) { throw new Error(f('ambiguous unwrapped union: %j', this)); } this._bucketIndices[bucket] = index; } }, this); } util.inherits(UnwrappedUnionType, UnionType); UnwrappedUnionType.prototype._getIndex = function (val) { var index = this._bucketIndices[getValueBucket(val)]; if (this._logicalBranches) { // Slower path, we must run the value through all logical types. index = this._getLogicalIndex(val, index); } return index; }; UnwrappedUnionType.prototype._getLogicalIndex = function (any, index) { var logicalBranches = this._logicalBranches; var i, l, branch; for (i = 0, l = logicalBranches.length; i < l; i++) { branch = logicalBranches[i]; if (branch.type._check(any)) { if (index === undefined) { index = branch.index; } else { // More than one branch matches the value so we aren't guaranteed to // infer the correct type. We throw rather than corrupt data. This can // be fixed by "tightening" the logical types. throw new Error('ambiguous conversion'); } } } return index; }; UnwrappedUnionType.prototype._check = function (val, flags, hook, path) { var index = this._getIndex(val); var b = index !== undefined; if (b) { return this._types[index]._check(val, flags, hook, path); } if (hook) { hook(val, this); } return b; }; UnwrappedUnionType.prototype._read = function (tap) { var index = tap.readLong(); var branchType = this._types[index]; if (branchType) { return branchType._read(tap); } else { throw new Error(f('invalid union index: %s', index)); } }; UnwrappedUnionType.prototype._write = function (tap, val) { var index = this._getIndex(val); if (index === undefined) { throwInvalidError(val, this); } tap.writeLong(index); if (val !== null) { this._types[index]._write(tap, val); } }; UnwrappedUnionType.prototype._update = function (resolver, type, opts) { // jshint -W083 // (The loop exits after the first function is created.) var i, l, typeResolver; for (i = 0, l = this._types.length; i < l; i++) { try { typeResolver = this._types[i].createResolver(type, opts); } catch (err) { continue; } resolver._read = function (tap) { return typeResolver._read(tap); }; return; } }; UnwrappedUnionType.prototype._copy = function (val, opts) { var coerce = opts && opts.coerce | 0; var wrap = opts && opts.wrap | 0; var index; if (wrap === 2) { // We are parsing a default, so always use the first branch's type. index = 0; } else { switch (coerce) { case 1: // Using the `coerceBuffers` option can cause corruption and erroneous // failures with unwrapped unions (in rare cases when the union also // contains a record which matches a buffer's JSON representation). if (isJsonBuffer(val) && this._bucketIndices.buffer !== undefined) { index = this._bucketIndices.buffer; } else { index = this._getIndex(val); } break; case 2: // Decoding from JSON, we must unwrap the value. if (val === null) { index = this._bucketIndices['null']; } else if (typeof val === 'object') { var keys = Object.keys(val); if (keys.length === 1) { index = this._branchIndices[keys[0]]; val = val[keys[0]]; } } break; default: index = this._getIndex(val); } if (index === undefined) { throwInvalidError(val, this); } } var type = this._types[index]; if (val === null || wrap === 3) { return type._copy(val, opts); } else { switch (coerce) { case 3: // Encoding to JSON, we wrap the value. var obj = {}; obj[type.getName(true)] = type._copy(val, opts); return obj; default: return type._copy(val, opts); } } }; UnwrappedUnionType.prototype.compare = function (val1, val2) { var index1 = this._getIndex(val1); var index2 = this._getIndex(val2); if (index1 === undefined) { throwInvalidError(val1, this); } else if (index2 === undefined) { throwInvalidError(val2, this); } else if (index1 === index2) { return this._types[index1].compare(val1, val2); } else { return utils.compare(index1, index2); } }; UnwrappedUnionType.prototype.getTypeName = function () { return 'union:unwrapped'; }; UnwrappedUnionType.prototype.random = function () { var index = RANDOM.nextInt(this._types.length); return this._types[index].random(); }; /** * Compatible union type. * * Values of this type are represented in memory similarly to their JSON * representation (i.e. inside an object with single key the name of the * contained type). * * This is not ideal, but is the most efficient way to unambiguously support * all unions. Here are a few reasons why the wrapping object is necessary: * * + Unions with multiple number types would have undefined behavior, unless * numbers are wrapped (either everywhere, leading to large performance and * convenience costs; or only when necessary inside unions, making it hard to * understand when numbers are wrapped or not). * + Fixed types would have to be wrapped to be distinguished from bytes. * + Using record's constructor names would work (after a slight change to use * the fully qualified name), but would mean that generic objects could no * longer be valid records (making it inconvenient to do simple things like * creating new records). * */ function WrappedUnionType(attrs, opts) { UnionType.call(this, attrs, opts); this._constructors = this._types.map(function (type) { // jshint -W054 var name = type.getName(true); if (name === 'null') { return null; } function ConstructorFunction(name) { return function Branch$(val) { this[`${name}`] = val; } } var constructor = ConstructorFunction(name); constructor.getBranchType = function() { return type; }; return constructor; }); } util.inherits(WrappedUnionType, UnionType); WrappedUnionType.prototype._check = function (val, flags, hook, path) { var b = false; if (val === null) { // Shortcut type lookup in this case. b = this._branchIndices['null'] !== undefined; } else if (typeof val == 'object') { var keys = Object.keys(val); if (keys.length === 1) { // We require a single key here to ensure that writes are correct and // efficient as soon as a record passes this check. var name = keys[0]; var index = this._branchIndices[name]; if (index !== undefined) { if (hook) { // Slow path. path.push(name); b = this._types[index]._check(val[name], flags, hook, path); path.pop(); return b; } else { return this._types[index]._check(val[name], flags); } } } } if (!b && hook) { hook(val, this); } return b; }; WrappedUnionType.prototype._read = function (tap) { var index = tap.readLong(); var Class = this._constructors[index]; if (Class) { return new Class(this._types[index]._read(tap)); } else if (Class === null) { return null; } else { throw new Error(f('invalid union index: %s', index)); } }; WrappedUnionType.prototype._write = function (tap, val) { var index, keys, name; if (val === null) { index = this._branchIndices['null']; if (index === undefined) { throwInvalidError(val, this); } tap.writeLong(index); } else { keys = Object.keys(val); if (keys.length === 1) { name = keys[0]; index = this._branchIndices[name]; } if (index === undefined) { throwInvalidError(val, this); } tap.writeLong(index); this._types[index]._write(tap, val[name]); } }; WrappedUnionType.prototype._update = function (resolver, type, opts) { // jshint -W083 // (The loop exits after the first function is created.) var i, l, typeResolver, Class; for (i = 0, l = this._types.length; i < l; i++) { try { typeResolver = this._types[i].createResolver(type, opts); } catch (err) { continue; } Class = this._constructors[i]; if (Class) { resolver._read = function (tap) { return new Class(typeResolver._read(tap)); }; } else { resolver._read = function () { return null; }; } return; } }; WrappedUnionType.prototype._copy = function (val, opts) { var wrap = opts && opts.wrap | 0; if (wrap === 2) { // Promote into first type (used for schema defaults). if (val === null && this._constructors[0] === null) { return null; } return new this._constructors[0](this._types[0]._copy(val, opts)); } if (val === null && this._branchIndices['null'] !== undefined) { return null; } var i, l, obj; if (typeof val == 'object') { var keys = Object.keys(val); if (keys.length === 1) { var name = keys[0]; i = this._branchIndices[name]; if (i === undefined && opts.qualifyNames) { // We are a bit more flexible than in `_check` here since we have // to deal with other serializers being less strict, so we fall // back to looking up unqualified names. var j, type; for (j = 0, l = this._types.length; j < l; j++) { type = this._types[j]; if (type._name && name === unqualify(type._name)) { i = j; break; } } } if (i !== undefined) { obj = this._types[i]._copy(val[name], opts); } } } if (wrap === 1 && obj === undefined) { // Try promoting into first match (convenience, slow). i = 0; l = this._types.length; while (i < l && obj === undefined) { try { obj = this._types[i]._copy(val, opts); } catch (err) { i++; } } } if (obj !== undefined) { return wrap === 3 ? obj : new this._constructors[i](obj); } throwInvalidError(val, this); }; WrappedUnionType.prototype.compare = function (val1, val2) { var name1 = val1 === null ? 'null' : Object.keys(val1)[0]; var name2 = val2 === null ? 'null' : Object.keys(val2)[0]; var index = this._branchIndices[name1]; if (name1 === name2) { return name1 === 'null' ? 0 : this._types[index].compare(val1[name1], val2[name1]); } else { return utils.compare(index, this._branchIndices[name2]); } }; WrappedUnionType.prototype.getTypeName = function () { return 'union:wrapped'; }; WrappedUnionType.prototype.random = function () { var index = RANDOM.nextInt(this._types.length); var Class = this._constructors[index]; if (!Class) { return null; } return new Class(this._types[index].random()); }; /** * Avro enum type. * * Represented as strings (with allowed values from the set of symbols). Using * integers would be a reasonable option, but the performance boost is arguably * offset by the legibility cost and the extra deviation from the JSON encoding * convention. * * An integer representation can still be used (e.g. for compatibility with * TypeScript `enum`s) by overriding the `EnumType` with a `LongType` (e.g. via * `parse`'s registry). * */ function EnumType(attrs, opts) { Type.call(this, attrs, opts); if (!Array.isArray(attrs.symbols) || !attrs.symbols.length) { throw new Error(f('invalid enum symbols: %j', attrs.symbols)); } this._symbols = attrs.symbols; this._indices = {}; this._symbols.forEach(function (symbol, i) { if (!isValidName(symbol)) { throw new Error(f('invalid %s symbol: %j', this, symbol)); } if (this._indices[symbol] !== undefined) { throw new Error(f('duplicate %s symbol: %j', this, symbol)); } this._indices[symbol] = i; }, this); } util.inherits(EnumType, Type); EnumType.prototype._check = function (val, flags, hook) { var b = this._indices[val] !== undefined; if (!b && hook) { hook(val, this); } return b; }; EnumType.prototype._read = function (tap) { var index = tap.readLong(); var symbol = this._symbols[index]; if (symbol === undefined) { throw new Error(f('invalid %s enum index: %s', this._name, index)); } return symbol; }; EnumType.prototype._skip = function (tap) { tap.skipLong(); }; EnumType.prototype._write = function (tap, val) { var index = this._indices[val]; if (index === undefined) { throwInvalidError(val, this); } tap.writeLong(index); }; EnumType.prototype._match = function (tap1, tap2) { return tap1.matchLong(tap2); }; EnumType.prototype.compare = function (val1, val2) { return utils.compare(this._indices[val1], this._indices[val2]); }; EnumType.prototype._update = function (resolver, type) { var symbols = this._symbols; if ( type.getTypeName() === 'enum' && (!type._name || ~getAliases(this).indexOf(type._name)) && type._symbols.every(function (s) { return ~symbols.indexOf(s); }) ) { resolver._symbols = type._symbols; resolver._read = type._read; } }; EnumType.prototype._copy = function (val) { this._check(val, undefined, throwInvalidError); return val; }; EnumType.prototype.getAliases = function () { return this._aliases; }; EnumType.prototype.getSymbols = function () { return this._symbols.slice(); }; EnumType.prototype.getTypeName = function () { return 'enum'; }; EnumType.prototype.random = function () { return RANDOM.choice(this._symbols); }; EnumType.prototype.toJSON = function () { return { name: this._name, type: this.getTypeName(), symbols: this._symbols, aliases: this._aliases }; }; /** * Avro fixed type. * * Represented simply as a `Buffer`. * */ function FixedType(attrs, opts) { Type.call(this, attrs, opts); if (attrs.size !== (attrs.size | 0) || attrs.size < 1) { throw new Error(f('invalid %s fixed size', this.getName(true))); } this._size = attrs.size | 0; } util.inherits(FixedType, Type); FixedType.prototype._check = function (val, flags, hook) { var b = Buffer.isBuffer(val) && val.length === this._size; if (!b && hook) { hook(val, this); } return b; }; FixedType.prototype._read = function (tap) { return tap.readFixed(this._size); }; FixedType.prototype._skip = function (tap) { tap.skipFixed(this._size); }; FixedType.prototype._write = function (tap, val) { if (!Buffer.isBuffer(val) || val.length !== this._size) { throwInvalidError(val, this); } tap.writeFixed(val, this._size); }; FixedType.prototype._match = function (tap1, tap2) { return tap1.matchFixed(tap2, this._size); }; FixedType.prototype.compare = Buffer.compare; FixedType.prototype._update = function (resolver, type) { if ( type.getTypeName() === 'fixed' && this._size === type._size && (!type._name || ~getAliases(this).indexOf(type._name)) ) { resolver._size = this._size; resolver._read = this._read; } }; FixedType.prototype._copy = BytesType.prototype._copy; FixedType.prototype.getAliases = function () { return this._aliases; }; FixedType.prototype.getSize = function () { return this._size; }; FixedType.prototype.getTypeName = function () { return 'fixed'; }; FixedType.prototype.random = function () { return RANDOM.nextBuffer(this._size); }; FixedType.prototype.toJSON = function () { return { name: this._name, type: this.getTypeName(), size: this._size, aliases: this._aliases }; }; /** * Avro map. * * Represented as vanilla objects. * */ function MapType(attrs, opts) { Type.call(this); if (!attrs.values) { throw new Error(f('missing map values: %j', attrs)); } this._values = createType(attrs.values, opts); // Addition by Edwin Elia var keys = attrs.keys; if (!keys) { keys = 'string'; } this._keys = createType(keys, opts); } util.inherits(MapType, Type); MapType.prototype._check = function (val, flags, hook, path) { if (!val || typeof val != 'object' || Array.isArray(val)) { if (hook) { hook(val, this); } return false; } var keys = Object.keys(val); var b = true; var i, l, j, key; if (hook) { // Slow path. j = path.length; path.push(''); for (i = 0, l = keys.length; i < l; i++) { key = path[j] = keys[i]; if (!this._values._check(val[key], flags, hook, path)) { b = false; } } path.pop(); } else { for (i = 0, l = keys.length; i < l; i++) { if (!this._values._check(val[keys[i]], flags)) { return false; } } } return b; }; MapType.prototype._read = function (tap) { var values = this._values; var val = {}; var n; while ((n = readArraySize(tap))) { while (n--) { var key = tap.readString(); val[key] = values._read(tap); } } return val; }; MapType.prototype._skip = function (tap) { var values = this._values; var len, n; while ((n = tap.readLong())) { if (n < 0) { len = tap.readLong(); tap.pos += len; } else { while (n--) { tap.skipString(); values._skip(tap); } } } }; MapType.prototype._write = function (tap, val) { if (!val || typeof val != 'object' || Array.isArray(val)) { throwInvalidError(val, this); } var values = this._values; var keys = Object.keys(val); var n = keys.length; var i, key; if (n) { tap.writeLong(n); for (i = 0; i < n; i++) { key = keys[i]; tap.writeString(key); values._write(tap, val[key]); } } tap.writeLong(0); }; MapType.prototype._match = function () { throw new Error('maps cannot be compared'); }; MapType.prototype._update = function (resolver, type, opts) { if (type.getTypeName() === 'map') { resolver._values = this._values.createResolver(type._values, opts); resolver._read = this._read; } }; MapType.prototype._copy = function (val, opts) { if (val && typeof val == 'object' && !Array.isArray(val)) { var values = this._values; var keys = Object.keys(val); var i, l, key; var copy = {}; for (i = 0, l = keys.length; i < l; i++) { key = keys[i]; copy[key] = values._copy(val[key], opts); } return copy; } throwInvalidError(val, this); }; MapType.prototype.compare = MapType.prototype._match; MapType.prototype.getTypeName = function () { return 'map'; }; MapType.prototype.getValuesType = function () { return this._values; }; // Addition by Edwin Elia MapType.prototype.getKeysType = function () { return this._keys; }; MapType.prototype.random = function () { var val = {}; var i, l; for (i = 0, l = RANDOM.nextInt(10); i < l; i++) { val[RANDOM.nextString(RANDOM.nextInt(20))] = this._values.random(); } return val; }; MapType.prototype.toJSON = function () { return {type: this.getTypeName(), values: this._values}; }; /** * Avro array. * * Represented as vanilla arrays. * */ function ArrayType(attrs, opts) { Type.call(this); if (!attrs.items) { throw new Error(f('missing array items: %j', attrs)); } this._items = createType(attrs.items, opts); } util.inherits(ArrayType, Type); ArrayType.prototype._check = function (val, flags, hook, path) { if (!Array.isArray(val)) { if (hook) { hook(val, this); } return false; } var b = true; var i, l, j; if (hook) { // Slow path. j = path.length; path.push(''); for (i = 0, l = val.length; i < l; i++) { path[j] = '' + i; if (!this._items._check(val[i], flags, hook, path)) { b = false; } } path.pop(); } else { for (i = 0, l = val.length; i < l; i++) { if (!this._items._check(val[i], flags)) { return false; } } } return b; }; ArrayType.prototype._read = function (tap) { var items = this._items; var val = []; var i, n; while ((n = tap.readLong())) { if (n < 0) { n = -n; tap.skipLong(); // Skip size. } for (i = 0; i < n; i++) { val[i] = items._read(tap); } } return val; }; ArrayType.prototype._skip = function (tap) { var len, n; while ((n = tap.readLong())) { if (n < 0) { len = tap.readLong(); tap.pos += len; } else { while (n--) { this._items._skip(tap); } } } }; ArrayType.prototype._write = function (tap, val) { if (!Array.isArray(val)) { throwInvalidError(val, this); } var n = val.length; var i; if (n) { tap.writeLong(n); for (i = 0; i < n; i++) { this._items._write(tap, val[i]); } } tap.writeLong(0); }; ArrayType.prototype._match = function (tap1, tap2) { var n1 = tap1.readLong(); var n2 = tap2.readLong(); var f; while (n1 && n2) { f = this._items._match(tap1, tap2); if (f) { return f; } if (!--n1) { n1 = readArraySize(tap1); } if (!--n2) { n2 = readArraySize(tap2); } } return utils.compare(n1, n2); }; ArrayType.prototype._update = function (resolver, type, opts) { if (type.getTypeName() === 'array') { resolver._items = this._items.createResolver(type._items, opts); resolver._read = this._read; } }; ArrayType.prototype._copy = function (val, opts) { if (!Array.isArray(val)) { throwInvalidError(val, this); } var items = new Array(val.length); var i, l; for (i = 0, l = val.length; i < l; i++) { items[i] = this._items._copy(val[i], opts); } return items; }; ArrayType.prototype.compare = function (val1, val2) { var n1 = val1.length; var n2 = val2.length; var i, l, f; for (i = 0, l = Math.min(n1, n2); i < l; i++) { if ((f = this._items.compare(val1[i], val2[i]))) { return f; } } return utils.compare(n1, n2); }; ArrayType.prototype.getItemsType = function () { return this._items; }; ArrayType.prototype.getTypeName = function () { return 'array'; }; ArrayType.prototype.random = function () { var arr = []; var i, l; for (i = 0, l = RANDOM.nextInt(10); i < l; i++) { arr.push(this._items.random()); } return arr; }; ArrayType.prototype.toJSON = function () { return {type: this.getTypeName(), items: this._items}; }; /** * Avro record. * * Values are represented as instances of a programmatically generated * constructor (similar to a "specific record"), available via the * `getRecordConstructor` method. This "specific record class" gives * significant speedups over using generics objects. * * Note that vanilla objects are still accepted as valid as long as their * fields match (this makes it much more convenient to do simple things like * update nested records). * * This type is also used for errors (similar, except for the extra `Error` * constructor call) and for messages (see comment below). * */ function RecordType(attrs, opts) { // Force creation of the options object in case we need to register this // record's name. opts = opts || {}; // Save the namespace to restore it as we leave this record's scope. var namespace = opts.namespace; if (attrs.namespace !== undefined) { opts.namespace = attrs.namespace; } else if (attrs.name) { // Fully qualified names' namespaces are used when no explicit namespace // attribute was specified. var match = /^(.*)\.[^.]+$/.exec(attrs.name); if (match) { opts.namespace = match[1]; } } Type.call(this, attrs, opts); if (!Array.isArray(attrs.fields)) { throw new Error(f('non-array record fields: %j', attrs.fields)); } if (utils.hasDuplicates(attrs.fields, function (f) { return f.name; })) { throw new Error(f('duplicate field name: %j', attrs.fields)); } this._fieldsMap = {}; this._fields = attrs.fields.map(function (f) { var field = new Field(f, opts); this._fieldsMap[field.getName()] = field; return field; }, this); this._isError = attrs.type === 'error'; this._constructor = this._createConstructor(); this._read = this._createReader(); this._skip = this._createSkipper(); this._write = this._createWriter(); this._check = this._createChecker(); opts.namespace = namespace; } util.inherits(RecordType, Type); RecordType.prototype._getConstructorName = function () { return this._name ? unqualify(this._name) : this._isError ? 'Error$' : 'Record$'; }; RecordType.prototype._createConstructor = function () { // jshint -W054 var ds = []; // Defaults. // Not calling `Error.captureStackTrace` because this wouldn't be compatible // with browsers other than Chrome. var i, l, field, name, getDefault; for (i = 0, l = this._fields.length; i < l; i++) { field = this._fields[i]; getDefault = field.getDefault; // Addition by Ajai // Allow hyphens to be part of field name (including enums) if (getDefault() !== undefined) { ds.push(getDefault); } } var self = this; function ConstructorFunction(oArgs) { var constructorName = self._getConstructorName(); var innerFunction = ({ [constructorName]: function (...iArgs) { if (self._isError) { Error.call(self); } self._fields.f