protons
Version:
Protobuf to ts transpiler
1,088 lines (1,077 loc) • 39.2 kB
JavaScript
/* eslint-disable max-depth */
/**
* @packageDocumentation
*
* `protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/).
*
* It transpiles code to TypeScript and supports BigInts for 64 bit types.
*
* The `protons` module contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution.
*
* Please ensure you declare them as the correct type of dependencies:
*
* ```console
* $ npm install --save-dev protons
* $ npm install --save protons-runtime
* ```
*
* ## Usage
*
* First generate your `.ts` files:
*
* ```console
* $ protons ./path/to/foo.proto ./path/to/output.ts
* ```
*
* Then run tsc over them as normal:
*
* ```console
* $ tsc
* ```
*
* In your code import the generated classes and use them to transform to/from bytes:
*
* ```js
* import { Foo } from './foo.js'
*
* const foo = {
* message: 'hello world'
* }
*
* const encoded = Foo.encode(foo)
* const decoded = Foo.decode(encoded)
*
* console.info(decoded.message)
* // 'hello world'
* ```
*
* ## Differences from protobuf.js
*
* This module uses the internal reader/writer from `protobuf.js` as it is highly optimised and there's no point reinventing the wheel.
*
* It does have one or two differences:
*
* 1. Supports `proto3` semantics only
* 2. All 64 bit values are represented as `BigInt`s and not `Long`s (e.g. `int64`, `uint64`, `sint64` etc)
* 3. Unset `optional` fields are set on the deserialized object forms as `undefined` instead of the default values
* 4. `singular` fields set to default values are not serialized and are set to default values when deserialized if not set - protobuf.js [diverges from the language guide](https://github.com/protobufjs/protobuf.js/issues/1468#issuecomment-745177012) around this feature
* 5. `map` fields can have keys of any type - protobufs.js [only supports strings](https://github.com/protobufjs/protobuf.js/issues/1203#issuecomment-488637338)
* 6. `map` fields are deserialized as ES6 `Map`s - protobuf.js uses `Object`s
*
* ## Extra features
*
* ### Limiting the size of repeated/map elements
*
* To protect decoders from malicious payloads, it's possible to limit the maximum size of repeated/map elements.
*
* You can either do this at compile time by using the [protons.options](https://github.com/protocolbuffers/protobuf/blob/6f1d88107f268b8ebdad6690d116e74c403e366e/docs/options.md?plain=1#L490-L493) extension:
*
* ```protobuf
* message MyMessage {
* // repeatedField cannot have more than 10 entries
* repeated uint32 repeatedField = 1 [(protons.options).limit = 10];
*
* // stringMap cannot have more than 10 keys
* map<string, string> stringMap = 2 [(protons.options).limit = 10];
* }
* ```
*
* Or at runtime by passing objects to the `.decode` function of your message:
*
* ```TypeScript
* const message = MyMessage.decode(buf, {
* limits: {
* repeatedField: 10,
* stringMap: 10
* }
* })
* ```
*
* #### Limiting repeating fields of nested messages at runtime
*
* Sub messages with repeating elements can be limited in a similar way:
*
* ```protobuf
* message SubMessage {
* repeated uint32 repeatedField = 1;
* }
*
* message MyMessage {
* SubMessage message = 1;
* }
* ```
*
* ```TypeScript
* const message = MyMessage.decode(buf, {
* limits: {
* messages: {
* repeatedField: 5 // the SubMessage can not have more than 5 repeatedField entries
* }
* }
* })
* ```
*
* #### Limiting repeating fields of repeating messages at runtime
*
* Sub messages defined in repeating elements can be limited by appending `$` to the field name in the runtime limit options:
*
* ```protobuf
* message SubMessage {
* repeated uint32 repeatedField = 1;
* }
*
* message MyMessage {
* repeated SubMessage messages = 1;
* }
* ```
*
* ```TypeScript
* const message = MyMessage.decode(buf, {
* limits: {
* messages: 5 // max 5x SubMessages
* messages$: {
* repeatedField: 5 // no SubMessage can have more than 5 repeatedField entries
* }
* }
* })
* ```
*
* #### Limiting repeating fields of map entries at runtime
*
* Repeating fields in map entries can be limited by appending `$value` to the field name in the runtime limit options:
*
* ```protobuf
* message SubMessage {
* repeated uint32 repeatedField = 1;
* }
*
* message MyMessage {
* map<string, SubMessage> messages = 1;
* }
* ```
*
* ```TypeScript
* const message = MyMessage.decode(buf, {
* limits: {
* messages: 5 // max 5x SubMessages in the map
* messages$value: {
* repeatedField: 5 // no SubMessage in the map can have more than 5 repeatedField entries
* }
* }
* })
* ```
*
* ### Overriding 64 bit types
*
* By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.
*
* Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility.
*
* It's possible to override the JavaScript type 64 bit fields will deserialize to:
*
* ```protobuf
* message MyMessage {
* repeated int64 bigintField = 1;
* repeated int64 numberField = 2 [jstype = JS_NUMBER];
* repeated int64 stringField = 3 [jstype = JS_STRING];
* }
* ```
*
* ```TypeScript
* const message = MyMessage.decode(buf)
*
* console.info(typeof message.bigintField) // bigint
* console.info(typeof message.numberField) // number
* console.info(typeof message.stringField) // string
* ```
*
* ## Missing features
*
* Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.
*/
import fs from 'fs/promises';
import path from 'path';
import { promisify } from 'util';
import { main as pbjs } from 'protobufjs-cli/pbjs.js';
import { NoMessagesFoundError, ParseError } from 'protons-runtime';
export var CODEC_TYPES;
(function (CODEC_TYPES) {
CODEC_TYPES[CODEC_TYPES["VARINT"] = 0] = "VARINT";
CODEC_TYPES[CODEC_TYPES["BIT64"] = 1] = "BIT64";
CODEC_TYPES[CODEC_TYPES["LENGTH_DELIMITED"] = 2] = "LENGTH_DELIMITED";
CODEC_TYPES[CODEC_TYPES["START_GROUP"] = 3] = "START_GROUP";
CODEC_TYPES[CODEC_TYPES["END_GROUP"] = 4] = "END_GROUP";
CODEC_TYPES[CODEC_TYPES["BIT32"] = 5] = "BIT32";
})(CODEC_TYPES || (CODEC_TYPES = {}));
function pathWithExtension(input, extension, outputDir) {
const output = outputDir ?? path.dirname(input);
return path.join(output, path.basename(input).split('.').slice(0, -1).join('.') + extension);
}
/**
* This will be removed in a future release
*
* @deprecated
*/
export class CodeError extends Error {
code;
constructor(message, code, options) {
super(message, options);
this.code = code;
}
}
const types = {
bool: 'boolean',
bytes: 'Uint8Array',
double: 'number',
fixed32: 'number',
fixed64: 'bigint',
float: 'number',
int32: 'number',
int64: 'bigint',
sfixed32: 'number',
sfixed64: 'bigint',
sint32: 'number',
sint64: 'bigint',
string: 'string',
uint32: 'number',
uint64: 'bigint'
};
const jsTypeOverrides = {
JS_NUMBER: 'number',
JS_STRING: 'string'
};
const encoderGenerators = {
bool: (val) => `w.bool(${val})`,
bytes: (val) => `w.bytes(${val})`,
double: (val) => `w.double(${val})`,
fixed32: (val) => `w.fixed32(${val})`,
fixed64: (val, jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return `w.fixed64Number(${val})`;
}
if (jsTypeOverride === 'string') {
return `w.fixed64String(${val})`;
}
return `w.fixed64(${val})`;
},
float: (val) => `w.float(${val})`,
int32: (val) => `w.int32(${val})`,
int64: (val, jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return `w.int64Number(${val})`;
}
if (jsTypeOverride === 'string') {
return `w.int64String(${val})`;
}
return `w.int64(${val})`;
},
sfixed32: (val) => `w.sfixed32(${val})`,
sfixed64: (val, jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return `w.sfixed64Number(${val})`;
}
if (jsTypeOverride === 'string') {
return `w.sfixed64String(${val})`;
}
return `w.sfixed64(${val})`;
},
sint32: (val) => `w.sint32(${val})`,
sint64: (val, jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return `w.sint64Number(${val})`;
}
if (jsTypeOverride === 'string') {
return `w.sint64String(${val})`;
}
return `w.sint64(${val})`;
},
string: (val) => `w.string(${val})`,
uint32: (val) => `w.uint32(${val})`,
uint64: (val, jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return `w.uint64Number(${val})`;
}
if (jsTypeOverride === 'string') {
return `w.uint64String(${val})`;
}
return `w.uint64(${val})`;
}
};
const decoderGenerators = {
bool: () => 'reader.bool()',
bytes: () => 'reader.bytes()',
double: () => 'reader.double()',
fixed32: () => 'reader.fixed32()',
fixed64: (jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return 'reader.fixed64Number()';
}
if (jsTypeOverride === 'string') {
return 'reader.fixed64String()';
}
return 'reader.fixed64()';
},
float: () => 'reader.float()',
int32: () => 'reader.int32()',
int64: (jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return 'reader.int64Number()';
}
if (jsTypeOverride === 'string') {
return 'reader.int64String()';
}
return 'reader.int64()';
},
sfixed32: () => 'reader.sfixed32()',
sfixed64: (jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return 'reader.sfixed64Number()';
}
if (jsTypeOverride === 'string') {
return 'reader.sfixed64String()';
}
return 'reader.sfixed64()';
},
sint32: () => 'reader.sint32()',
sint64: (jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return 'reader.sint64Number()';
}
if (jsTypeOverride === 'string') {
return 'reader.sint64String()';
}
return 'reader.sint64()';
},
string: () => 'reader.string()',
uint32: () => 'reader.uint32()',
uint64: (jsTypeOverride) => {
if (jsTypeOverride === 'number') {
return 'reader.uint64Number()';
}
if (jsTypeOverride === 'string') {
return 'reader.uint64String()';
}
return 'reader.uint64()';
}
};
const defaultValueGenerators = {
bool: () => 'false',
bytes: () => 'uint8ArrayAlloc(0)',
double: () => '0',
fixed32: () => '0',
fixed64: () => '0n',
float: () => '0',
int32: () => '0',
int64: () => '0n',
sfixed32: () => '0',
sfixed64: () => '0n',
sint32: () => '0',
sint64: () => '0n',
string: () => "''",
uint32: () => '0',
uint64: () => '0n'
};
const defaultValueGeneratorsJsTypeOverrides = {
number: () => '0',
string: () => "''"
};
const defaultValueTestGenerators = {
bool: (field) => `(${field} != null && ${field} !== false)`,
bytes: (field) => `(${field} != null && ${field}.byteLength > 0)`,
double: (field) => `(${field} != null && ${field} !== 0)`,
fixed32: (field) => `(${field} != null && ${field} !== 0)`,
fixed64: (field) => `(${field} != null && ${field} !== 0n)`,
float: (field) => `(${field} != null && ${field} !== 0)`,
int32: (field) => `(${field} != null && ${field} !== 0)`,
int64: (field) => `(${field} != null && ${field} !== 0n)`,
sfixed32: (field) => `(${field} != null && ${field} !== 0)`,
sfixed64: (field) => `(${field} != null && ${field} !== 0n)`,
sint32: (field) => `(${field} != null && ${field} !== 0)`,
sint64: (field) => `(${field} != null && ${field} !== 0n)`,
string: (field) => `(${field} != null && ${field} !== '')`,
uint32: (field) => `(${field} != null && ${field} !== 0)`,
uint64: (field) => `(${field} != null && ${field} !== 0n)`
};
const defaultValueTestGeneratorsJsTypeOverrides = {
number: (field) => `(${field} != null && ${field} !== 0)`,
string: (field) => `(${field} != null && ${field} !== '')`
};
function findJsTypeOverride(defaultType, fieldDef) {
if (fieldDef.options?.jstype != null && jsTypeOverrides[fieldDef.options?.jstype] != null) {
if (!['int64', 'uint64', 'sint64', 'fixed64', 'sfixed64'].includes(defaultType)) {
throw new Error(`jstype is only allowed on int64, uint64, sint64, fixed64 or sfixed64 fields - got "${defaultType}"`);
}
return jsTypeOverrides[fieldDef.options?.jstype];
}
}
function findJsTypeName(typeName, classDef, moduleDef, fieldDef) {
const override = findJsTypeOverride(typeName, fieldDef);
if (override != null) {
return override;
}
if (types[typeName] != null) {
return types[typeName];
}
if (isEnumDef(classDef)) {
throw new Error('Could not find type in enum');
}
if (classDef.nested?.[typeName] != null) {
return `${classDef.fullName}.${typeName}`;
}
if (classDef.parent != null) {
return findJsTypeName(typeName, classDef.parent, moduleDef, fieldDef);
}
if (moduleDef.globals[typeName] != null) {
return typeName;
}
throw new Error(`Could not resolve type name "${typeName}"`);
}
function findDef(typeName, classDef, moduleDef) {
if (isEnumDef(classDef)) {
throw new Error('Could not find type in enum');
}
if (classDef.nested?.[typeName] != null) {
return classDef.nested?.[typeName];
}
if (classDef.parent != null) {
return findDef(typeName, classDef.parent, moduleDef);
}
if (moduleDef.globals[typeName] != null) {
return moduleDef.globals[typeName];
}
throw new Error(`Could not resolve type name "${typeName}"`);
}
function createDefaultObject(fields, messageDef, moduleDef) {
const output = Object.entries(fields)
.map(([name, fieldDef]) => {
if (fieldDef.map) {
return `${name}: new Map<${types[fieldDef.keyType ?? 'string']}, ${types[fieldDef.valueType]}>()`;
}
if (fieldDef.repeated) {
return `${name}: []`;
}
if (fieldDef.optional) {
return '';
}
const type = fieldDef.type;
let defaultValue;
let defaultValueGenerator = defaultValueGenerators[type];
if (defaultValueGenerator != null) {
const jsTypeOverride = findJsTypeOverride(type, fieldDef);
if (jsTypeOverride != null && defaultValueGeneratorsJsTypeOverrides[jsTypeOverride] != null) {
defaultValueGenerator = defaultValueGeneratorsJsTypeOverrides[jsTypeOverride];
}
if (type === 'bytes') {
moduleDef.addImport('uint8arrays/alloc', 'alloc', 'uint8ArrayAlloc');
}
defaultValue = defaultValueGenerator();
}
else {
const def = findDef(fieldDef.type, messageDef, moduleDef);
if (isEnumDef(def)) {
// select lowest-value enum - should be 0 but it's not guaranteed
const val = Object.entries(def.values)
.sort((a, b) => {
if (a[1] < b[1]) {
return 1;
}
if (a[1] > b[1]) {
return -1;
}
return 0;
})
.pop();
if (val == null) {
throw new Error(`Could not find default enum value for ${def.fullName}`);
}
defaultValue = `${def.name}.${val[0]}`;
}
else {
defaultValue = 'undefined';
}
}
return `${name}: ${defaultValue}`;
})
.filter(Boolean)
.join(',\n ');
if (output !== '') {
return `
${output}
`;
}
return '';
}
const encoders = {
bool: 'bool',
bytes: 'bytes',
double: 'double',
fixed32: 'fixed32',
fixed64: 'fixed64',
float: 'float',
int32: 'int32',
int64: 'int64',
sfixed32: 'sfixed32',
sfixed64: 'sfixed64',
sint32: 'sint32',
sint64: 'sint64',
string: 'string',
uint32: 'uint32',
uint64: 'uint64'
};
const codecTypes = {
bool: CODEC_TYPES.VARINT,
bytes: CODEC_TYPES.LENGTH_DELIMITED,
double: CODEC_TYPES.BIT64,
enum: CODEC_TYPES.VARINT,
fixed32: CODEC_TYPES.BIT32,
fixed64: CODEC_TYPES.BIT64,
float: CODEC_TYPES.BIT32,
int32: CODEC_TYPES.VARINT,
int64: CODEC_TYPES.VARINT,
message: CODEC_TYPES.LENGTH_DELIMITED,
sfixed32: CODEC_TYPES.BIT32,
sfixed64: CODEC_TYPES.BIT64,
sint32: CODEC_TYPES.VARINT,
sint64: CODEC_TYPES.VARINT,
string: CODEC_TYPES.LENGTH_DELIMITED,
uint32: CODEC_TYPES.VARINT,
uint64: CODEC_TYPES.VARINT
};
function isEnumDef(obj) {
return obj.values != null;
}
function defineFields(fields, messageDef, moduleDef) {
return Object.entries(fields).map(([fieldName, fieldDef]) => {
if (fieldDef.map) {
return `${fieldName}: Map<${findJsTypeName(fieldDef.keyType ?? 'string', messageDef, moduleDef, fieldDef)}, ${findJsTypeName(fieldDef.valueType, messageDef, moduleDef, fieldDef)}>`;
}
return `${fieldName}${fieldDef.optional ? '?' : ''}: ${findJsTypeName(fieldDef.type, messageDef, moduleDef, fieldDef)}${fieldDef.repeated ? '[]' : ''}`;
});
}
function compileMessage(messageDef, moduleDef, flags) {
if (isEnumDef(messageDef)) {
moduleDef.addImport('protons-runtime', 'enumeration');
// check that the enum def values start from 0
if (Object.values(messageDef.values)[0] !== 0) {
const message = `enum ${messageDef.name} does not contain a value that maps to zero as it's first element, this is required in proto3 - see https://protobuf.dev/programming-guides/proto3/#enum`;
if (flags?.strict === true) {
throw new ParseError(message);
}
else {
// eslint-disable-next-line no-console
console.info(`[WARN] ${message}`);
}
}
return `
export enum ${messageDef.name} {
${Object.keys(messageDef.values).map(name => {
return `${name} = '${name}'`;
}).join(',\n ').trim()}
}
enum __${messageDef.name}Values {
${Object.entries(messageDef.values).map(([name, value]) => {
return `${name} = ${value}`;
}).join(',\n ').trim()}
}
export namespace ${messageDef.name} {
export const codec = (): Codec<${messageDef.name}> => {
return enumeration<${messageDef.name}>(__${messageDef.name}Values)
}
}`.trim();
}
let nested = '';
if (messageDef.nested != null) {
nested = '\n';
nested += Object.values(messageDef.nested)
.map(def => compileMessage(def, moduleDef, flags).trim())
.join('\n\n')
.split('\n')
.map(line => line.trim() === '' ? '' : ` ${line}`)
.join('\n');
}
const fields = messageDef.fields ?? {};
// import relevant modules
moduleDef.addImport('protons-runtime', 'encodeMessage');
moduleDef.addImport('protons-runtime', 'decodeMessage');
moduleDef.addImport('protons-runtime', 'message');
moduleDef.addTypeImport('protons-runtime', 'Codec');
moduleDef.addTypeImport('protons-runtime', 'DecodeOptions');
moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList');
const interfaceFields = defineFields(fields, messageDef, moduleDef)
.join('\n ')
.trim();
let interfaceDef = '';
let interfaceCodecDef = '';
if (interfaceFields === '') {
interfaceDef = `
export interface ${messageDef.name} {}`;
}
else {
interfaceDef = `
export interface ${messageDef.name} {
${defineFields(fields, messageDef, moduleDef)
.join('\n ')
.trim()}
}`;
}
const encodeFields = Object.entries(fields)
.map(([name, fieldDef]) => {
let codec = encoders[fieldDef.type];
let type = fieldDef.map ? 'message' : fieldDef.type;
let typeName = '';
if (codec == null) {
if (fieldDef.enum) {
moduleDef.addImport('protons-runtime', 'enumeration');
type = 'enum';
}
else {
moduleDef.addImport('protons-runtime', 'message');
type = 'message';
}
typeName = findJsTypeName(fieldDef.type, messageDef, moduleDef, fieldDef);
codec = `${typeName}.codec()`;
}
let valueTest = `obj.${name} != null`;
if (fieldDef.map) {
valueTest = `obj.${name} != null && obj.${name}.size !== 0`;
}
else if (!fieldDef.optional && !fieldDef.repeated && !fieldDef.proto2Required) {
let defaultValueTestGenerator = defaultValueTestGenerators[type];
// proto3 singular fields should only be written out if they are not the default value
if (defaultValueTestGenerator != null) {
const jsTypeOverride = findJsTypeOverride(type, fieldDef);
if (jsTypeOverride != null && defaultValueTestGeneratorsJsTypeOverrides[jsTypeOverride] != null) {
defaultValueTestGenerator = defaultValueTestGeneratorsJsTypeOverrides[jsTypeOverride];
}
valueTest = `${defaultValueTestGenerator(`obj.${name}`)}`;
}
else if (type === 'enum') {
// handle enums
const def = findDef(fieldDef.type, messageDef, moduleDef);
if (!isEnumDef(def)) {
throw new Error(`${fieldDef.type} was not enum def`);
}
valueTest = `obj.${name} != null`;
// singular enums default to 0, but enums can be defined without a 0
// value which is against the proto3 spec but is tolerated
if (Object.values(def.values)[0] === 0) {
valueTest += ` && __${fieldDef.type}Values[obj.${name}] !== 0`;
}
}
}
function createWriteField(valueVar) {
const id = (fieldDef.id << 3) | codecTypes[type];
if (fieldDef.enum) {
const def = findDef(fieldDef.type, messageDef, moduleDef);
if (!isEnumDef(def)) {
throw new Error(`${fieldDef.type} was not enum def`);
}
}
let writeField = () => {
const encoderGenerator = encoderGenerators[type];
const jsTypeOverride = findJsTypeOverride(type, fieldDef);
return `w.uint32(${id})
${encoderGenerator == null ? `${codec}.encode(${valueVar}, w)` : encoderGenerator(valueVar, jsTypeOverride)}`;
};
if (type === 'message') {
// message fields are only written if they have values. But if a message
// is part of a repeated field, and consists of only default values it
// won't be written, so write a zero-length buffer if that's the case
writeField = () => `w.uint32(${id})
${typeName}.codec().encode(${valueVar}, w)`;
}
return writeField;
}
let writeField = createWriteField(`obj.${name}`);
if (fieldDef.repeated) {
if (fieldDef.map) {
writeField = () => `
for (const [key, value] of obj.${name}.entries()) {
${createWriteField('{ key, value }')()
.split('\n')
.map(s => {
const trimmed = s.trim();
return trimmed === '' ? trimmed : ` ${s}`;
})
.join('\n')}
}
`.trim();
}
else {
writeField = () => `
for (const value of obj.${name}) {
${createWriteField('value')()
.split('\n')
.map(s => {
const trimmed = s.trim();
return trimmed === '' ? trimmed : ` ${s}`;
})
.join('\n')}
}
`.trim();
}
}
return `
if (${valueTest}) {
${writeField()}
}`;
}).join('\n');
const decodeFields = Object.entries(fields)
.map(([fieldName, fieldDef]) => {
function createReadField(fieldName, fieldDef) {
let codec = encoders[fieldDef.type];
let type = fieldDef.type;
if (codec == null) {
if (fieldDef.enum) {
moduleDef.addImport('protons-runtime', 'enumeration');
type = 'enum';
}
else {
moduleDef.addImport('protons-runtime', 'message');
type = 'message';
}
const typeName = findJsTypeName(fieldDef.type, messageDef, moduleDef, fieldDef);
codec = `${typeName}.codec()`;
}
// override setting type on js object
const jsTypeOverride = findJsTypeOverride(fieldDef.type, fieldDef);
let fieldOpts = '';
if (fieldDef.message) {
let suffix = '';
if (fieldDef.repeated) {
suffix = '$';
}
fieldOpts = `, {
limits: opts.limits?.${fieldName}${suffix}
}`;
}
if (fieldDef.map) {
fieldOpts = `, {
limits: {
value: opts.limits?.${fieldName}$value
}
}`;
// do not pass limit opts to map value types that are enums or
// primitives - only support messages
if (types[fieldDef.valueType] != null) {
// primmitive type
fieldOpts = '';
}
else {
const valueType = findDef(fieldDef.valueType, messageDef, moduleDef);
if (isEnumDef(valueType)) {
// enum type
fieldOpts = '';
}
}
}
const parseValue = `${decoderGenerators[type] == null
? `${codec}.decode(reader${type === 'message'
? `, reader.uint32()${fieldOpts}`
: ''})`
: decoderGenerators[type](jsTypeOverride)}`;
if (fieldDef.map) {
moduleDef.addImport('protons-runtime', 'MaxSizeError');
let limit = `
if (opts.limits?.${fieldName} != null && obj.${fieldName}.size === opts.limits.${fieldName}) {
throw new MaxSizeError('Decode error - map field "${fieldName}" had too many elements')
}
`;
if (fieldDef.lengthLimit != null) {
limit += `
if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
throw new MaxSizeError('Decode error - map field "${fieldName}" had too many elements')
}
`;
}
return `case ${fieldDef.id}: {${limit}
const entry = ${parseValue}
obj.${fieldName}.set(entry.key, entry.value)
break
}`;
}
else if (fieldDef.repeated) {
moduleDef.addImport('protons-runtime', 'MaxLengthError');
let limit = `
if (opts.limits?.${fieldName} != null && obj.${fieldName}.length === opts.limits.${fieldName}) {
throw new MaxLengthError('Decode error - map field "${fieldName}" had too many elements')
}
`;
if (fieldDef.lengthLimit != null) {
limit += `
if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
throw new MaxLengthError('Decode error - repeated field "${fieldName}" had too many elements')
}
`;
}
return `case ${fieldDef.id}: {${limit}
obj.${fieldName}.push(${parseValue})
break
}`;
}
return `case ${fieldDef.id}: {
obj.${fieldName} = ${parseValue}
break
}`;
}
return createReadField(fieldName, fieldDef);
})
.join('\n ');
interfaceCodecDef = `
let _codec: Codec<${messageDef.name}>
export const codec = (): Codec<${messageDef.name}> => {
if (_codec == null) {
_codec = message<${messageDef.name}>((obj, w, opts = {}) => {
if (opts.lengthDelimited !== false) {
w.fork()
}
${encodeFields === '' ? '' : `${encodeFields}\n`}
if (opts.lengthDelimited !== false) {
w.ldelim()
}
}, (reader, length, opts = {}) => {
const obj: any = {${createDefaultObject(fields, messageDef, moduleDef)}}
const end = length == null ? reader.len : reader.pos + length
while (reader.pos < end) {
const tag = reader.uint32()
switch (tag >>> 3) {${decodeFields === '' ? '' : `\n ${decodeFields}`}
default: {
reader.skipType(tag & 7)
break
}
}
}
return obj
})
}
return _codec
}
export const encode = (obj: Partial<${messageDef.name}>): Uint8Array => {
return encodeMessage(obj, ${messageDef.name}.codec())
}
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<${messageDef.name}>): ${messageDef.name} => {
return decodeMessage(buf, ${messageDef.name}.codec(), opts)
}`;
return `
${interfaceDef}
export namespace ${messageDef.name} {
${`${nested}${nested !== '' && interfaceCodecDef !== '' ? '\n' : ''}${interfaceCodecDef}`.trim()}
}
`.trimStart();
}
class ModuleDef {
imports;
types;
compiled;
globals;
constructor() {
this.imports = new Map();
this.types = new Set();
this.compiled = [];
this.globals = {};
}
addImport(module, symbol, alias) {
const defs = this._findDefs(module);
for (const def of defs) {
// check if we already have a definition for this symbol
if (def.symbol === symbol) {
if (alias !== def.alias) {
throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`);
}
// if it was a type before it's not now
def.type = false;
return;
}
}
defs.push({
symbol,
alias,
type: false
});
}
addTypeImport(module, symbol, alias) {
const defs = this._findDefs(module);
for (const def of defs) {
// check if we already have a definition for this symbol
if (def.symbol === symbol) {
if (alias !== def.alias) {
throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`);
}
return;
}
}
defs.push({
symbol,
alias,
type: true
});
}
_findDefs(module) {
let defs = this.imports.get(module);
if (defs == null) {
defs = [];
this.imports.set(module, defs);
}
return defs;
}
}
function defineModule(def, flags) {
const moduleDef = new ModuleDef();
const defs = def.nested;
if (defs == null) {
throw new NoMessagesFoundError('No top-level messages found in protobuf');
}
function defineMessage(defs, parent, flags) {
for (const className of Object.keys(defs)) {
const classDef = defs[className];
classDef.name = className;
classDef.parent = parent;
classDef.fullName = parent == null ? className : `${parent.fullName}.${className}`;
if (classDef.nested != null) {
defineMessage(classDef.nested, classDef);
}
if (classDef.fields != null) {
for (const name of Object.keys(classDef.fields)) {
const fieldDef = classDef.fields[name];
fieldDef.repeated = fieldDef.rule === 'repeated';
fieldDef.optional = !fieldDef.repeated && fieldDef.options?.proto3_optional === true;
fieldDef.map = fieldDef.keyType != null;
fieldDef.lengthLimit = fieldDef.options?.['(protons.options).limit'];
fieldDef.proto2Required = false;
if (fieldDef.rule === 'required') {
const message = `field "${name}" is required, this is not allowed in proto3. Please convert your proto2 definitions to proto3 - see https://github.com/ipfs/protons/wiki/Required-fields-and-protobuf-3`;
if (flags?.strict === true) {
throw new ParseError(message);
}
else {
fieldDef.proto2Required = true;
// eslint-disable-next-line no-console
console.info(`[WARN] ${message}`);
}
}
}
}
if (parent == null) {
moduleDef.globals[className] = classDef;
}
}
}
function updateTypes(defs, parent) {
for (const className of Object.keys(defs)) {
const classDef = defs[className];
if (classDef.nested != null) {
updateTypes(classDef.nested, classDef);
}
if (classDef.fields != null) {
for (const name of Object.keys(classDef.fields)) {
const fieldDef = classDef.fields[name];
if (types[fieldDef.type] == null) {
const def = findDef(fieldDef.type, classDef, moduleDef);
fieldDef.enum = isEnumDef(def);
fieldDef.message = !fieldDef.enum;
if (fieldDef.message && !fieldDef.repeated) {
// the default type for a message is unset so they are always optional
// https://developers.google.com/protocol-buffers/docs/proto3#default
fieldDef.optional = true;
}
}
}
}
}
}
defineMessage(defs, undefined, flags);
// set enum/message fields now all messages have been defined
updateTypes(defs);
for (const className of Object.keys(defs)) {
const classDef = defs[className];
moduleDef.compiled.push(compileMessage(classDef, moduleDef, flags));
}
return moduleDef;
}
export async function generate(source, flags) {
// convert .protobuf to .json
const json = await promisify(pbjs)([
'-t', 'json',
...(flags.path ?? []).map(p => ['--path', path.isAbsolute(p) ? p : path.resolve(process.cwd(), p)]).flat(),
source
]);
if (json == null) {
throw new Error(`Could not convert ${source} to intermediate JSON format`);
}
const def = JSON.parse(json);
for (const [className, classDef] of Object.entries(def.nested ?? {})) {
for (const [fieldName, fieldDef] of Object.entries(classDef.fields ?? {})) {
if (fieldDef.keyType == null) {
continue;
}
// https://developers.google.com/protocol-buffers/docs/proto3#backwards_compatibility
const mapEntryType = `${className}$${fieldName}Entry`;
classDef.nested = classDef.nested ?? {};
classDef.nested[mapEntryType] = {
fields: {
key: {
type: fieldDef.keyType,
id: 1
},
value: {
type: fieldDef.type,
id: 2
}
}
};
fieldDef.valueType = fieldDef.type;
fieldDef.type = mapEntryType;
fieldDef.rule = 'repeated';
}
}
const moduleDef = defineModule(def, flags);
const ignores = [
'/* eslint-disable import/export */',
'/* eslint-disable complexity */',
'/* eslint-disable @typescript-eslint/no-namespace */',
'/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */',
'/* eslint-disable @typescript-eslint/no-empty-interface */'
];
const imports = [];
const importedModules = Array.from([...moduleDef.imports.entries()])
.sort((a, b) => {
return a[0].localeCompare(b[0]);
})
.sort((a, b) => {
const aAllTypes = a[1].reduce((acc, curr) => {
return acc && curr.type;
}, true);
const bAllTypes = b[1].reduce((acc, curr) => {
return acc && curr.type;
}, true);
if (aAllTypes && !bAllTypes) {
return 1;
}
if (!aAllTypes && bAllTypes) {
return -1;
}
return 0;
});
for (const imp of importedModules) {
const allTypes = imp[1].reduce((acc, curr) => {
return acc && curr.type;
}, true);
const symbols = imp[1].sort((a, b) => {
return a.symbol.localeCompare(b.symbol);
}).map(imp => {
return `${!allTypes && imp.type ? 'type ' : ''}${imp.symbol}${imp.alias != null ? ` as ${imp.alias}` : ''}`;
}).join(', ');
imports.push(`import ${allTypes ? 'type ' : ''}{ ${symbols} } from '${imp[0]}'`);
}
const lines = [
...ignores,
'',
...imports,
'',
...moduleDef.compiled
];
const content = lines.join('\n').trim();
const outputPath = pathWithExtension(source, '.ts', flags.output);
await fs.writeFile(outputPath, content + '\n');
}
//# sourceMappingURL=index.js.map