@huggingface/transformers
Version:
State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!
1,601 lines (1,573 loc) • 1.73 MB
JavaScript
import * as __WEBPACK_EXTERNAL_MODULE_node_fs_75ed2103__ from "node:fs";
import * as __WEBPACK_EXTERNAL_MODULE_node_path_02319fef__ from "node:path";
import * as __WEBPACK_EXTERNAL_MODULE_node_url_da953c0c__ from "node:url";
import * as __WEBPACK_EXTERNAL_MODULE_onnxruntime_common_82b39e9f__ from "onnxruntime-common";
import * as __WEBPACK_EXTERNAL_MODULE_onnxruntime_node_6a60201e__ from "onnxruntime-node";
import * as __WEBPACK_EXTERNAL_MODULE_sharp__ from "sharp";
/******/ var __webpack_modules__ = ({
/***/ "node:fs":
/*!**************************!*\
!*** external "node:fs" ***!
\**************************/
/***/ ((module) => {
module.exports = __WEBPACK_EXTERNAL_MODULE_node_fs_75ed2103__;
/***/ }),
/***/ "node:path":
/*!****************************!*\
!*** external "node:path" ***!
\****************************/
/***/ ((module) => {
module.exports = __WEBPACK_EXTERNAL_MODULE_node_path_02319fef__;
/***/ }),
/***/ "node:url":
/*!***************************!*\
!*** external "node:url" ***!
\***************************/
/***/ ((module) => {
module.exports = __WEBPACK_EXTERNAL_MODULE_node_url_da953c0c__;
/***/ }),
/***/ "onnxruntime-common":
/*!*************************************!*\
!*** external "onnxruntime-common" ***!
\*************************************/
/***/ ((module) => {
module.exports = __WEBPACK_EXTERNAL_MODULE_onnxruntime_common_82b39e9f__;
/***/ }),
/***/ "onnxruntime-node":
/*!***********************************!*\
!*** external "onnxruntime-node" ***!
\***********************************/
/***/ ((module) => {
module.exports = __WEBPACK_EXTERNAL_MODULE_onnxruntime_node_6a60201e__;
/***/ }),
/***/ "sharp":
/*!************************!*\
!*** external "sharp" ***!
\************************/
/***/ ((module) => {
module.exports = __WEBPACK_EXTERNAL_MODULE_sharp__;
/***/ }),
/***/ "?8b6b":
/*!*********************************!*\
!*** onnxruntime-web (ignored) ***!
\*********************************/
/***/ (() => {
/* (ignored) */
/***/ }),
/***/ "./node_modules/@huggingface/jinja/dist/index.js":
/*!*******************************************************!*\
!*** ./node_modules/@huggingface/jinja/dist/index.js ***!
\*******************************************************/
/***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __webpack_require__) => {
__webpack_require__.r(__webpack_exports__);
/* harmony export */ __webpack_require__.d(__webpack_exports__, {
/* harmony export */ Environment: () => (/* binding */ Environment),
/* harmony export */ Interpreter: () => (/* binding */ Interpreter),
/* harmony export */ Template: () => (/* binding */ Template),
/* harmony export */ parse: () => (/* binding */ parse),
/* harmony export */ tokenize: () => (/* binding */ tokenize)
/* harmony export */ });
// src/lexer.ts
var TOKEN_TYPES = Object.freeze({
Text: "Text",
// The text between Jinja statements or expressions
NumericLiteral: "NumericLiteral",
// e.g., 123, 1.0
StringLiteral: "StringLiteral",
// 'string'
Identifier: "Identifier",
// Variables, functions, statements, booleans, etc.
Equals: "Equals",
// =
OpenParen: "OpenParen",
// (
CloseParen: "CloseParen",
// )
OpenStatement: "OpenStatement",
// {%
CloseStatement: "CloseStatement",
// %}
OpenExpression: "OpenExpression",
// {{
CloseExpression: "CloseExpression",
// }}
OpenSquareBracket: "OpenSquareBracket",
// [
CloseSquareBracket: "CloseSquareBracket",
// ]
OpenCurlyBracket: "OpenCurlyBracket",
// {
CloseCurlyBracket: "CloseCurlyBracket",
// }
Comma: "Comma",
// ,
Dot: "Dot",
// .
Colon: "Colon",
// :
Pipe: "Pipe",
// |
CallOperator: "CallOperator",
// ()
AdditiveBinaryOperator: "AdditiveBinaryOperator",
// + - ~
MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator",
// * / %
ComparisonBinaryOperator: "ComparisonBinaryOperator",
// < > <= >= == !=
UnaryOperator: "UnaryOperator",
// ! - +
Comment: "Comment"
// {# ... #}
});
var Token = class {
/**
* Constructs a new Token.
* @param {string} value The raw value as seen inside the source code.
* @param {TokenType} type The type of token.
*/
constructor(value, type) {
this.value = value;
this.type = type;
}
};
function isWord(char) {
return /\w/.test(char);
}
function isInteger(char) {
return /[0-9]/.test(char);
}
var ORDERED_MAPPING_TABLE = [
// Control sequences
["{%", TOKEN_TYPES.OpenStatement],
["%}", TOKEN_TYPES.CloseStatement],
["{{", TOKEN_TYPES.OpenExpression],
["}}", TOKEN_TYPES.CloseExpression],
// Single character tokens
["(", TOKEN_TYPES.OpenParen],
[")", TOKEN_TYPES.CloseParen],
["{", TOKEN_TYPES.OpenCurlyBracket],
["}", TOKEN_TYPES.CloseCurlyBracket],
["[", TOKEN_TYPES.OpenSquareBracket],
["]", TOKEN_TYPES.CloseSquareBracket],
[",", TOKEN_TYPES.Comma],
[".", TOKEN_TYPES.Dot],
[":", TOKEN_TYPES.Colon],
["|", TOKEN_TYPES.Pipe],
// Comparison operators
["<=", TOKEN_TYPES.ComparisonBinaryOperator],
[">=", TOKEN_TYPES.ComparisonBinaryOperator],
["==", TOKEN_TYPES.ComparisonBinaryOperator],
["!=", TOKEN_TYPES.ComparisonBinaryOperator],
["<", TOKEN_TYPES.ComparisonBinaryOperator],
[">", TOKEN_TYPES.ComparisonBinaryOperator],
// Arithmetic operators
["+", TOKEN_TYPES.AdditiveBinaryOperator],
["-", TOKEN_TYPES.AdditiveBinaryOperator],
["~", TOKEN_TYPES.AdditiveBinaryOperator],
["*", TOKEN_TYPES.MultiplicativeBinaryOperator],
["/", TOKEN_TYPES.MultiplicativeBinaryOperator],
["%", TOKEN_TYPES.MultiplicativeBinaryOperator],
// Assignment operator
["=", TOKEN_TYPES.Equals]
];
var ESCAPE_CHARACTERS = /* @__PURE__ */ new Map([
["n", "\n"],
// New line
["t", " "],
// Horizontal tab
["r", "\r"],
// Carriage return
["b", "\b"],
// Backspace
["f", "\f"],
// Form feed
["v", "\v"],
// Vertical tab
["'", "'"],
// Single quote
['"', '"'],
// Double quote
["\\", "\\"]
// Backslash
]);
function preprocess(template, options = {}) {
if (template.endsWith("\n")) {
template = template.slice(0, -1);
}
if (options.lstrip_blocks) {
template = template.replace(/^[ \t]*({[#%-])/gm, "$1");
}
if (options.trim_blocks) {
template = template.replace(/([#%-]})\n/g, "$1");
}
return template.replace(/-%}\s*/g, "%}").replace(/\s*{%-/g, "{%").replace(/-}}\s*/g, "}}").replace(/\s*{{-/g, "{{").replace(/-#}\s*/g, "#}").replace(/\s*{#-/g, "{#").replace(/{%\s*(end)?generation\s*%}/gs, "");
}
function tokenize(source, options = {}) {
const tokens = [];
const src = preprocess(source, options);
let cursorPosition = 0;
let curlyBracketDepth = 0;
const consumeWhile = (predicate) => {
let str = "";
while (predicate(src[cursorPosition])) {
if (src[cursorPosition] === "\\") {
++cursorPosition;
if (cursorPosition >= src.length)
throw new SyntaxError("Unexpected end of input");
const escaped = src[cursorPosition++];
const unescaped = ESCAPE_CHARACTERS.get(escaped);
if (unescaped === void 0) {
throw new SyntaxError(`Unexpected escaped character: ${escaped}`);
}
str += unescaped;
continue;
}
str += src[cursorPosition++];
if (cursorPosition >= src.length)
throw new SyntaxError("Unexpected end of input");
}
return str;
};
main:
while (cursorPosition < src.length) {
const lastTokenType = tokens.at(-1)?.type;
if (lastTokenType === void 0 || lastTokenType === TOKEN_TYPES.CloseStatement || lastTokenType === TOKEN_TYPES.CloseExpression || lastTokenType === TOKEN_TYPES.Comment) {
let text = "";
while (cursorPosition < src.length && // Keep going until we hit the next Jinja statement or expression
!(src[cursorPosition] === "{" && (src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{" || src[cursorPosition + 1] === "#"))) {
text += src[cursorPosition++];
}
if (text.length > 0) {
tokens.push(new Token(text, TOKEN_TYPES.Text));
continue;
}
}
if (src[cursorPosition] === "{" && src[cursorPosition + 1] === "#") {
cursorPosition += 2;
let comment = "";
while (src[cursorPosition] !== "#" || src[cursorPosition + 1] !== "}") {
if (cursorPosition + 2 >= src.length) {
throw new SyntaxError("Missing end of comment tag");
}
comment += src[cursorPosition++];
}
tokens.push(new Token(comment, TOKEN_TYPES.Comment));
cursorPosition += 2;
continue;
}
consumeWhile((char2) => /\s/.test(char2));
const char = src[cursorPosition];
if (char === "-" || char === "+") {
const lastTokenType2 = tokens.at(-1)?.type;
if (lastTokenType2 === TOKEN_TYPES.Text || lastTokenType2 === void 0) {
throw new SyntaxError(`Unexpected character: ${char}`);
}
switch (lastTokenType2) {
case TOKEN_TYPES.Identifier:
case TOKEN_TYPES.NumericLiteral:
case TOKEN_TYPES.StringLiteral:
case TOKEN_TYPES.CloseParen:
case TOKEN_TYPES.CloseSquareBracket:
break;
default: {
++cursorPosition;
const num = consumeWhile(isInteger);
tokens.push(
new Token(`${char}${num}`, num.length > 0 ? TOKEN_TYPES.NumericLiteral : TOKEN_TYPES.UnaryOperator)
);
continue;
}
}
}
for (const [seq, type] of ORDERED_MAPPING_TABLE) {
if (seq === "}}" && curlyBracketDepth > 0) {
continue;
}
const slice2 = src.slice(cursorPosition, cursorPosition + seq.length);
if (slice2 === seq) {
tokens.push(new Token(seq, type));
if (type === TOKEN_TYPES.OpenExpression) {
curlyBracketDepth = 0;
} else if (type === TOKEN_TYPES.OpenCurlyBracket) {
++curlyBracketDepth;
} else if (type === TOKEN_TYPES.CloseCurlyBracket) {
--curlyBracketDepth;
}
cursorPosition += seq.length;
continue main;
}
}
if (char === "'" || char === '"') {
++cursorPosition;
const str = consumeWhile((c) => c !== char);
tokens.push(new Token(str, TOKEN_TYPES.StringLiteral));
++cursorPosition;
continue;
}
if (isInteger(char)) {
let num = consumeWhile(isInteger);
if (src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) {
++cursorPosition;
const frac = consumeWhile(isInteger);
num = `${num}.${frac}`;
}
tokens.push(new Token(num, TOKEN_TYPES.NumericLiteral));
continue;
}
if (isWord(char)) {
const word = consumeWhile(isWord);
tokens.push(new Token(word, TOKEN_TYPES.Identifier));
continue;
}
throw new SyntaxError(`Unexpected character: ${char}`);
}
return tokens;
}
// src/ast.ts
var Statement = class {
type = "Statement";
};
var Program = class extends Statement {
constructor(body) {
super();
this.body = body;
}
type = "Program";
};
var If = class extends Statement {
constructor(test, body, alternate) {
super();
this.test = test;
this.body = body;
this.alternate = alternate;
}
type = "If";
};
var For = class extends Statement {
constructor(loopvar, iterable, body, defaultBlock) {
super();
this.loopvar = loopvar;
this.iterable = iterable;
this.body = body;
this.defaultBlock = defaultBlock;
}
type = "For";
};
var Break = class extends Statement {
type = "Break";
};
var Continue = class extends Statement {
type = "Continue";
};
var SetStatement = class extends Statement {
constructor(assignee, value, body) {
super();
this.assignee = assignee;
this.value = value;
this.body = body;
}
type = "Set";
};
var Macro = class extends Statement {
constructor(name, args, body) {
super();
this.name = name;
this.args = args;
this.body = body;
}
type = "Macro";
};
var Comment = class extends Statement {
constructor(value) {
super();
this.value = value;
}
type = "Comment";
};
var Expression = class extends Statement {
type = "Expression";
};
var MemberExpression = class extends Expression {
constructor(object, property, computed) {
super();
this.object = object;
this.property = property;
this.computed = computed;
}
type = "MemberExpression";
};
var CallExpression = class extends Expression {
constructor(callee, args) {
super();
this.callee = callee;
this.args = args;
}
type = "CallExpression";
};
var Identifier = class extends Expression {
/**
* @param {string} value The name of the identifier
*/
constructor(value) {
super();
this.value = value;
}
type = "Identifier";
};
var Literal = class extends Expression {
constructor(value) {
super();
this.value = value;
}
type = "Literal";
};
var IntegerLiteral = class extends Literal {
type = "IntegerLiteral";
};
var FloatLiteral = class extends Literal {
type = "FloatLiteral";
};
var StringLiteral = class extends Literal {
type = "StringLiteral";
};
var ArrayLiteral = class extends Literal {
type = "ArrayLiteral";
};
var TupleLiteral = class extends Literal {
type = "TupleLiteral";
};
var ObjectLiteral = class extends Literal {
type = "ObjectLiteral";
};
var BinaryExpression = class extends Expression {
constructor(operator, left, right) {
super();
this.operator = operator;
this.left = left;
this.right = right;
}
type = "BinaryExpression";
};
var FilterExpression = class extends Expression {
constructor(operand, filter) {
super();
this.operand = operand;
this.filter = filter;
}
type = "FilterExpression";
};
var FilterStatement = class extends Statement {
constructor(filter, body) {
super();
this.filter = filter;
this.body = body;
}
type = "FilterStatement";
};
var SelectExpression = class extends Expression {
constructor(lhs, test) {
super();
this.lhs = lhs;
this.test = test;
}
type = "SelectExpression";
};
var TestExpression = class extends Expression {
constructor(operand, negate, test) {
super();
this.operand = operand;
this.negate = negate;
this.test = test;
}
type = "TestExpression";
};
var UnaryExpression = class extends Expression {
constructor(operator, argument) {
super();
this.operator = operator;
this.argument = argument;
}
type = "UnaryExpression";
};
var SliceExpression = class extends Expression {
constructor(start = void 0, stop = void 0, step = void 0) {
super();
this.start = start;
this.stop = stop;
this.step = step;
}
type = "SliceExpression";
};
var KeywordArgumentExpression = class extends Expression {
constructor(key, value) {
super();
this.key = key;
this.value = value;
}
type = "KeywordArgumentExpression";
};
var SpreadExpression = class extends Expression {
constructor(argument) {
super();
this.argument = argument;
}
type = "SpreadExpression";
};
var CallStatement = class extends Statement {
constructor(call, callerArgs, body) {
super();
this.call = call;
this.callerArgs = callerArgs;
this.body = body;
}
type = "CallStatement";
};
var Ternary = class extends Expression {
constructor(condition, trueExpr, falseExpr) {
super();
this.condition = condition;
this.trueExpr = trueExpr;
this.falseExpr = falseExpr;
}
type = "Ternary";
};
// src/parser.ts
function parse(tokens) {
const program = new Program([]);
let current = 0;
function expect(type, error) {
const prev = tokens[current++];
if (!prev || prev.type !== type) {
throw new Error(`Parser Error: ${error}. ${prev.type} !== ${type}.`);
}
return prev;
}
function expectIdentifier(name) {
if (!isIdentifier(name)) {
throw new SyntaxError(`Expected ${name}`);
}
++current;
}
function parseAny() {
switch (tokens[current].type) {
case TOKEN_TYPES.Comment:
return new Comment(tokens[current++].value);
case TOKEN_TYPES.Text:
return parseText();
case TOKEN_TYPES.OpenStatement:
return parseJinjaStatement();
case TOKEN_TYPES.OpenExpression:
return parseJinjaExpression();
default:
throw new SyntaxError(`Unexpected token type: ${tokens[current].type}`);
}
}
function is(...types) {
return current + types.length <= tokens.length && types.every((type, i) => type === tokens[current + i].type);
}
function isStatement(...names) {
return tokens[current]?.type === TOKEN_TYPES.OpenStatement && tokens[current + 1]?.type === TOKEN_TYPES.Identifier && names.includes(tokens[current + 1]?.value);
}
function isIdentifier(...names) {
return current + names.length <= tokens.length && names.every((name, i) => tokens[current + i].type === "Identifier" && name === tokens[current + i].value);
}
function parseText() {
return new StringLiteral(expect(TOKEN_TYPES.Text, "Expected text token").value);
}
function parseJinjaStatement() {
expect(TOKEN_TYPES.OpenStatement, "Expected opening statement token");
if (tokens[current].type !== TOKEN_TYPES.Identifier) {
throw new SyntaxError(`Unknown statement, got ${tokens[current].type}`);
}
const name = tokens[current].value;
let result;
switch (name) {
case "set":
++current;
result = parseSetStatement();
break;
case "if":
++current;
result = parseIfStatement();
expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
expectIdentifier("endif");
expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
break;
case "macro":
++current;
result = parseMacroStatement();
expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
expectIdentifier("endmacro");
expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
break;
case "for":
++current;
result = parseForStatement();
expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
expectIdentifier("endfor");
expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
break;
case "call": {
++current;
let callerArgs = null;
if (is(TOKEN_TYPES.OpenParen)) {
callerArgs = parseArgs();
}
const callee = parsePrimaryExpression();
if (callee.type !== "Identifier") {
throw new SyntaxError(`Expected identifier following call statement`);
}
const callArgs = parseArgs();
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
const body = [];
while (!isStatement("endcall")) {
body.push(parseAny());
}
expect(TOKEN_TYPES.OpenStatement, "Expected '{%'");
expectIdentifier("endcall");
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
const callExpr = new CallExpression(callee, callArgs);
result = new CallStatement(callExpr, callerArgs, body);
break;
}
case "break":
++current;
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
result = new Break();
break;
case "continue":
++current;
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
result = new Continue();
break;
case "filter": {
++current;
let filterNode = parsePrimaryExpression();
if (filterNode instanceof Identifier && is(TOKEN_TYPES.OpenParen)) {
filterNode = parseCallExpression(filterNode);
}
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
const filterBody = [];
while (!isStatement("endfilter")) {
filterBody.push(parseAny());
}
expect(TOKEN_TYPES.OpenStatement, "Expected '{%'");
expectIdentifier("endfilter");
expect(TOKEN_TYPES.CloseStatement, "Expected '%}'");
result = new FilterStatement(filterNode, filterBody);
break;
}
default:
throw new SyntaxError(`Unknown statement type: ${name}`);
}
return result;
}
function parseJinjaExpression() {
expect(TOKEN_TYPES.OpenExpression, "Expected opening expression token");
const result = parseExpression();
expect(TOKEN_TYPES.CloseExpression, "Expected closing expression token");
return result;
}
function parseSetStatement() {
const left = parseExpressionSequence();
let value = null;
const body = [];
if (is(TOKEN_TYPES.Equals)) {
++current;
value = parseExpressionSequence();
} else {
expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
while (!isStatement("endset")) {
body.push(parseAny());
}
expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
expectIdentifier("endset");
}
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
return new SetStatement(left, value, body);
}
function parseIfStatement() {
const test = parseExpression();
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
const body = [];
const alternate = [];
while (!isStatement("elif", "else", "endif")) {
body.push(parseAny());
}
if (isStatement("elif")) {
++current;
++current;
const result = parseIfStatement();
alternate.push(result);
} else if (isStatement("else")) {
++current;
++current;
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
while (!isStatement("endif")) {
alternate.push(parseAny());
}
}
return new If(test, body, alternate);
}
function parseMacroStatement() {
const name = parsePrimaryExpression();
if (name.type !== "Identifier") {
throw new SyntaxError(`Expected identifier following macro statement`);
}
const args = parseArgs();
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
const body = [];
while (!isStatement("endmacro")) {
body.push(parseAny());
}
return new Macro(name, args, body);
}
function parseExpressionSequence(primary = false) {
const fn = primary ? parsePrimaryExpression : parseExpression;
const expressions = [fn()];
const isTuple = is(TOKEN_TYPES.Comma);
while (isTuple) {
++current;
expressions.push(fn());
if (!is(TOKEN_TYPES.Comma)) {
break;
}
}
return isTuple ? new TupleLiteral(expressions) : expressions[0];
}
function parseForStatement() {
const loopVariable = parseExpressionSequence(true);
if (!(loopVariable instanceof Identifier || loopVariable instanceof TupleLiteral)) {
throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${loopVariable.type} instead`);
}
if (!isIdentifier("in")) {
throw new SyntaxError("Expected `in` keyword following loop variable");
}
++current;
const iterable = parseExpression();
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
const body = [];
while (!isStatement("endfor", "else")) {
body.push(parseAny());
}
const alternative = [];
if (isStatement("else")) {
++current;
++current;
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
while (!isStatement("endfor")) {
alternative.push(parseAny());
}
}
return new For(loopVariable, iterable, body, alternative);
}
function parseExpression() {
return parseIfExpression();
}
function parseIfExpression() {
const a = parseLogicalOrExpression();
if (isIdentifier("if")) {
++current;
const test = parseLogicalOrExpression();
if (isIdentifier("else")) {
++current;
const falseExpr = parseIfExpression();
return new Ternary(test, a, falseExpr);
} else {
return new SelectExpression(a, test);
}
}
return a;
}
function parseLogicalOrExpression() {
let left = parseLogicalAndExpression();
while (isIdentifier("or")) {
const operator = tokens[current];
++current;
const right = parseLogicalAndExpression();
left = new BinaryExpression(operator, left, right);
}
return left;
}
function parseLogicalAndExpression() {
let left = parseLogicalNegationExpression();
while (isIdentifier("and")) {
const operator = tokens[current];
++current;
const right = parseLogicalNegationExpression();
left = new BinaryExpression(operator, left, right);
}
return left;
}
function parseLogicalNegationExpression() {
let right;
while (isIdentifier("not")) {
const operator = tokens[current];
++current;
const arg = parseLogicalNegationExpression();
right = new UnaryExpression(operator, arg);
}
return right ?? parseComparisonExpression();
}
function parseComparisonExpression() {
let left = parseAdditiveExpression();
while (true) {
let operator;
if (isIdentifier("not", "in")) {
operator = new Token("not in", TOKEN_TYPES.Identifier);
current += 2;
} else if (isIdentifier("in")) {
operator = tokens[current++];
} else if (is(TOKEN_TYPES.ComparisonBinaryOperator)) {
operator = tokens[current++];
} else {
break;
}
const right = parseAdditiveExpression();
left = new BinaryExpression(operator, left, right);
}
return left;
}
function parseAdditiveExpression() {
let left = parseMultiplicativeExpression();
while (is(TOKEN_TYPES.AdditiveBinaryOperator)) {
const operator = tokens[current];
++current;
const right = parseMultiplicativeExpression();
left = new BinaryExpression(operator, left, right);
}
return left;
}
function parseCallMemberExpression() {
const member = parseMemberExpression(parsePrimaryExpression());
if (is(TOKEN_TYPES.OpenParen)) {
return parseCallExpression(member);
}
return member;
}
function parseCallExpression(callee) {
let expression = new CallExpression(callee, parseArgs());
expression = parseMemberExpression(expression);
if (is(TOKEN_TYPES.OpenParen)) {
expression = parseCallExpression(expression);
}
return expression;
}
function parseArgs() {
expect(TOKEN_TYPES.OpenParen, "Expected opening parenthesis for arguments list");
const args = parseArgumentsList();
expect(TOKEN_TYPES.CloseParen, "Expected closing parenthesis for arguments list");
return args;
}
function parseArgumentsList() {
const args = [];
while (!is(TOKEN_TYPES.CloseParen)) {
let argument;
if (tokens[current].type === TOKEN_TYPES.MultiplicativeBinaryOperator && tokens[current].value === "*") {
++current;
const expr = parseExpression();
argument = new SpreadExpression(expr);
} else {
argument = parseExpression();
if (is(TOKEN_TYPES.Equals)) {
++current;
if (!(argument instanceof Identifier)) {
throw new SyntaxError(`Expected identifier for keyword argument`);
}
const value = parseExpression();
argument = new KeywordArgumentExpression(argument, value);
}
}
args.push(argument);
if (is(TOKEN_TYPES.Comma)) {
++current;
}
}
return args;
}
function parseMemberExpressionArgumentsList() {
const slices = [];
let isSlice = false;
while (!is(TOKEN_TYPES.CloseSquareBracket)) {
if (is(TOKEN_TYPES.Colon)) {
slices.push(void 0);
++current;
isSlice = true;
} else {
slices.push(parseExpression());
if (is(TOKEN_TYPES.Colon)) {
++current;
isSlice = true;
}
}
}
if (slices.length === 0) {
throw new SyntaxError(`Expected at least one argument for member/slice expression`);
}
if (isSlice) {
if (slices.length > 3) {
throw new SyntaxError(`Expected 0-3 arguments for slice expression`);
}
return new SliceExpression(...slices);
}
return slices[0];
}
function parseMemberExpression(object) {
while (is(TOKEN_TYPES.Dot) || is(TOKEN_TYPES.OpenSquareBracket)) {
const operator = tokens[current];
++current;
let property;
const computed = operator.type === TOKEN_TYPES.OpenSquareBracket;
if (computed) {
property = parseMemberExpressionArgumentsList();
expect(TOKEN_TYPES.CloseSquareBracket, "Expected closing square bracket");
} else {
property = parsePrimaryExpression();
if (property.type !== "Identifier") {
throw new SyntaxError(`Expected identifier following dot operator`);
}
}
object = new MemberExpression(object, property, computed);
}
return object;
}
function parseMultiplicativeExpression() {
let left = parseTestExpression();
while (is(TOKEN_TYPES.MultiplicativeBinaryOperator)) {
const operator = tokens[current++];
const right = parseTestExpression();
left = new BinaryExpression(operator, left, right);
}
return left;
}
function parseTestExpression() {
let operand = parseFilterExpression();
while (isIdentifier("is")) {
++current;
const negate = isIdentifier("not");
if (negate) {
++current;
}
const filter = parsePrimaryExpression();
if (!(filter instanceof Identifier)) {
throw new SyntaxError(`Expected identifier for the test`);
}
operand = new TestExpression(operand, negate, filter);
}
return operand;
}
function parseFilterExpression() {
let operand = parseCallMemberExpression();
while (is(TOKEN_TYPES.Pipe)) {
++current;
let filter = parsePrimaryExpression();
if (!(filter instanceof Identifier)) {
throw new SyntaxError(`Expected identifier for the filter`);
}
if (is(TOKEN_TYPES.OpenParen)) {
filter = parseCallExpression(filter);
}
operand = new FilterExpression(operand, filter);
}
return operand;
}
function parsePrimaryExpression() {
const token = tokens[current++];
switch (token.type) {
case TOKEN_TYPES.NumericLiteral: {
const num = token.value;
return num.includes(".") ? new FloatLiteral(Number(num)) : new IntegerLiteral(Number(num));
}
case TOKEN_TYPES.StringLiteral: {
let value = token.value;
while (is(TOKEN_TYPES.StringLiteral)) {
value += tokens[current++].value;
}
return new StringLiteral(value);
}
case TOKEN_TYPES.Identifier:
return new Identifier(token.value);
case TOKEN_TYPES.OpenParen: {
const expression = parseExpressionSequence();
expect(TOKEN_TYPES.CloseParen, "Expected closing parenthesis, got ${tokens[current].type} instead.");
return expression;
}
case TOKEN_TYPES.OpenSquareBracket: {
const values = [];
while (!is(TOKEN_TYPES.CloseSquareBracket)) {
values.push(parseExpression());
if (is(TOKEN_TYPES.Comma)) {
++current;
}
}
++current;
return new ArrayLiteral(values);
}
case TOKEN_TYPES.OpenCurlyBracket: {
const values = /* @__PURE__ */ new Map();
while (!is(TOKEN_TYPES.CloseCurlyBracket)) {
const key = parseExpression();
expect(TOKEN_TYPES.Colon, "Expected colon between key and value in object literal");
const value = parseExpression();
values.set(key, value);
if (is(TOKEN_TYPES.Comma)) {
++current;
}
}
++current;
return new ObjectLiteral(values);
}
default:
throw new SyntaxError(`Unexpected token: ${token.type}`);
}
}
while (current < tokens.length) {
program.body.push(parseAny());
}
return program;
}
// src/utils.ts
function range(start, stop, step = 1) {
if (stop === void 0) {
stop = start;
start = 0;
}
const result = [];
for (let i = start; i < stop; i += step) {
result.push(i);
}
return result;
}
function slice(array, start, stop, step = 1) {
const direction = Math.sign(step);
if (direction >= 0) {
start = (start ??= 0) < 0 ? Math.max(array.length + start, 0) : Math.min(start, array.length);
stop = (stop ??= array.length) < 0 ? Math.max(array.length + stop, 0) : Math.min(stop, array.length);
} else {
start = (start ??= array.length - 1) < 0 ? Math.max(array.length + start, -1) : Math.min(start, array.length - 1);
stop = (stop ??= -1) < -1 ? Math.max(array.length + stop, -1) : Math.min(stop, array.length - 1);
}
const result = [];
for (let i = start; direction * i < direction * stop; i += step) {
result.push(array[i]);
}
return result;
}
function titleCase(value) {
return value.replace(/\b\w/g, (c) => c.toUpperCase());
}
function strftime_now(format2) {
return strftime(/* @__PURE__ */ new Date(), format2);
}
function strftime(date, format2) {
const monthFormatterLong = new Intl.DateTimeFormat(void 0, { month: "long" });
const monthFormatterShort = new Intl.DateTimeFormat(void 0, { month: "short" });
const pad2 = (n) => n < 10 ? "0" + n : n.toString();
return format2.replace(/%[YmdbBHM%]/g, (token) => {
switch (token) {
case "%Y":
return date.getFullYear().toString();
case "%m":
return pad2(date.getMonth() + 1);
case "%d":
return pad2(date.getDate());
case "%b":
return monthFormatterShort.format(date);
case "%B":
return monthFormatterLong.format(date);
case "%H":
return pad2(date.getHours());
case "%M":
return pad2(date.getMinutes());
case "%%":
return "%";
default:
return token;
}
});
}
function escapeRegExp(s) {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function replace(str, oldvalue, newvalue, count) {
if (count === 0)
return str;
let remaining = count == null || count < 0 ? Infinity : count;
const pattern = oldvalue.length === 0 ? new RegExp("(?=)", "gu") : new RegExp(escapeRegExp(oldvalue), "gu");
return str.replaceAll(pattern, (match) => {
if (remaining > 0) {
--remaining;
return newvalue;
}
return match;
});
}
// src/runtime.ts
var BreakControl = class extends Error {
};
var ContinueControl = class extends Error {
};
var RuntimeValue = class {
type = "RuntimeValue";
value;
/**
* A collection of built-in functions for this type.
*/
builtins = /* @__PURE__ */ new Map();
/**
* Creates a new RuntimeValue.
*/
constructor(value = void 0) {
this.value = value;
}
/**
* Determines truthiness or falsiness of the runtime value.
* This function should be overridden by subclasses if it has custom truthiness criteria.
* @returns {BooleanValue} BooleanValue(true) if the value is truthy, BooleanValue(false) otherwise.
*/
__bool__() {
return new BooleanValue(!!this.value);
}
toString() {
return String(this.value);
}
};
var IntegerValue = class extends RuntimeValue {
type = "IntegerValue";
};
var FloatValue = class extends RuntimeValue {
type = "FloatValue";
toString() {
return this.value % 1 === 0 ? this.value.toFixed(1) : this.value.toString();
}
};
var StringValue = class extends RuntimeValue {
type = "StringValue";
builtins = /* @__PURE__ */ new Map([
[
"upper",
new FunctionValue(() => {
return new StringValue(this.value.toUpperCase());
})
],
[
"lower",
new FunctionValue(() => {
return new StringValue(this.value.toLowerCase());
})
],
[
"strip",
new FunctionValue(() => {
return new StringValue(this.value.trim());
})
],
[
"title",
new FunctionValue(() => {
return new StringValue(titleCase(this.value));
})
],
[
"capitalize",
new FunctionValue(() => {
return new StringValue(this.value.charAt(0).toUpperCase() + this.value.slice(1));
})
],
["length", new IntegerValue(this.value.length)],
[
"rstrip",
new FunctionValue(() => {
return new StringValue(this.value.trimEnd());
})
],
[
"lstrip",
new FunctionValue(() => {
return new StringValue(this.value.trimStart());
})
],
[
"startswith",
new FunctionValue((args) => {
if (args.length === 0) {
throw new Error("startswith() requires at least one argument");
}
const pattern = args[0];
if (pattern instanceof StringValue) {
return new BooleanValue(this.value.startsWith(pattern.value));
} else if (pattern instanceof ArrayValue) {
for (const item of pattern.value) {
if (!(item instanceof StringValue)) {
throw new Error("startswith() tuple elements must be strings");
}
if (this.value.startsWith(item.value)) {
return new BooleanValue(true);
}
}
return new BooleanValue(false);
}
throw new Error("startswith() argument must be a string or tuple of strings");
})
],
[
"endswith",
new FunctionValue((args) => {
if (args.length === 0) {
throw new Error("endswith() requires at least one argument");
}
const pattern = args[0];
if (pattern instanceof StringValue) {
return new BooleanValue(this.value.endsWith(pattern.value));
} else if (pattern instanceof ArrayValue) {
for (const item of pattern.value) {
if (!(item instanceof StringValue)) {
throw new Error("endswith() tuple elements must be strings");
}
if (this.value.endsWith(item.value)) {
return new BooleanValue(true);
}
}
return new BooleanValue(false);
}
throw new Error("endswith() argument must be a string or tuple of strings");
})
],
[
"split",
// follows Python's `str.split(sep=None, maxsplit=-1)` function behavior
// https://docs.python.org/3.13/library/stdtypes.html#str.split
new FunctionValue((args) => {
const sep = args[0] ?? new NullValue();
if (!(sep instanceof StringValue || sep instanceof NullValue)) {
throw new Error("sep argument must be a string or null");
}
const maxsplit = args[1] ?? new IntegerValue(-1);
if (!(maxsplit instanceof IntegerValue)) {
throw new Error("maxsplit argument must be a number");
}
let result = [];
if (sep instanceof NullValue) {
const text = this.value.trimStart();
for (const { 0: match, index } of text.matchAll(/\S+/g)) {
if (maxsplit.value !== -1 && result.length >= maxsplit.value && index !== void 0) {
result.push(match + text.slice(index + match.length));
break;
}
result.push(match);
}
} else {
if (sep.value === "") {
throw new Error("empty separator");
}
result = this.value.split(sep.value);
if (maxsplit.value !== -1 && result.length > maxsplit.value) {
result.push(result.splice(maxsplit.value).join(sep.value));
}
}
return new ArrayValue(result.map((part) => new StringValue(part)));
})
],
[
"replace",
new FunctionValue((args) => {
if (args.length < 2) {
throw new Error("replace() requires at least two arguments");
}
const oldValue = args[0];
const newValue = args[1];
if (!(oldValue instanceof StringValue && newValue instanceof StringValue)) {
throw new Error("replace() arguments must be strings");
}
let count;
if (args.length > 2) {
if (args[2].type === "KeywordArgumentsValue") {
count = args[2].value.get("count") ?? new NullValue();
} else {
count = args[2];
}
} else {
count = new NullValue();
}
if (!(count instanceof IntegerValue || count instanceof NullValue)) {
throw new Error("replace() count argument must be a number or null");
}
return new StringValue(replace(this.value, oldValue.value, newValue.value, count.value));
})
]
]);
};
var BooleanValue = class extends RuntimeValue {
type = "BooleanValue";
};
var ObjectValue = class extends RuntimeValue {
type = "ObjectValue";
/**
* NOTE: necessary to override since all JavaScript arrays are considered truthy,
* while only non-empty Python arrays are consider truthy.
*
* e.g.,
* - JavaScript: {} && 5 -> 5
* - Python: {} and 5 -> {}
*/
__bool__() {
return new BooleanValue(this.value.size > 0);
}
builtins = /* @__PURE__ */ new Map([
[
"get",
new FunctionValue(([key, defaultValue]) => {
if (!(key instanceof StringValue)) {
throw new Error(`Object key must be a string: got ${key.type}`);
}
return this.value.get(key.value) ?? defaultValue ?? new NullValue();
})
],
["items", new FunctionValue(() => this.items())],
["keys", new FunctionValue(() => this.keys())],
["values", new FunctionValue(() => this.values())]
]);
items() {
return new ArrayValue(
Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value]))
);
}
keys() {
return new ArrayValue(Array.from(this.value.keys()).map((key) => new StringValue(key)));
}
values() {
return new ArrayValue(Array.from(this.value.values()));
}
};
var KeywordArgumentsValue = class extends ObjectValue {
type = "KeywordArgumentsValue";
};
var ArrayValue = class extends RuntimeValue {
type = "ArrayValue";
builtins = /* @__PURE__ */ new Map([["length", new IntegerValue(this.value.length)]]);
/**
* NOTE: necessary to override since all JavaScript arrays are considered truthy,
* while only non-empty Python arrays are consider truthy.
*
* e.g.,
* - JavaScript: [] && 5 -> 5
* - Python: [] and 5 -> []
*/
__bool__() {
return new BooleanValue(this.value.length > 0);
}
};
var TupleValue = class extends ArrayValue {
type = "TupleValue";
};
var FunctionValue = class extends RuntimeValue {
type = "FunctionValue";
};
var NullValue = class extends RuntimeValue {
type = "NullValue";
};
var UndefinedValue = class extends RuntimeValue {
type = "UndefinedValue";
};
var Environment = class {
constructor(parent) {
this.parent = parent;
}
/**
* The variables declared in this environment.
*/
variables = /* @__PURE__ */ new Map([
[
"namespace",
new FunctionValue((args) => {
if (args.length === 0) {
return new ObjectValue(/* @__PURE__ */ new Map());
}
if (args.length !== 1 || !(args[0] instanceof ObjectValue)) {
throw new Error("`namespace` expects either zero arguments or a single object argument");
}
return args[0];
})
]
]);
/**
* The tests available in this environment.
*/
tests = /* @__PURE__ */ new Map([
["boolean", (operand) => operand.type === "BooleanValue"],
["callable", (operand) => operand instanceof FunctionValue],
[
"odd",
(operand) => {
if (!(operand instanceof IntegerValue)) {
throw new Error(`cannot odd on ${operand.type}`);
}
return operand.value % 2 !== 0;
}
],
[
"even",
(operand) => {
if (!(operand instanceof IntegerValue)) {
throw new Error(`cannot even on ${operand.type}`);
}
return operand.value % 2 === 0;
}
],
["false", (operand) => operand.type === "BooleanValue" && !operand.value],
["true", (operand) => operand.type === "BooleanValue" && operand.value],
["none", (operand) => operand.type === "NullValue"],
["string", (operand) => operand.type === "StringValue"],
["number", (operand) => operand instanceof IntegerValue || operand instanceof FloatValue],
["integer", (operand) => operand instanceof IntegerValue],
["iterable", (operand) => operand.type === "ArrayValue" || operand.type === "StringValue"],
["mapping", (operand) => operand.type === "ObjectValue"],
[
"lower",
(operand) => {
const str = operand.value;
return operand.type === "StringValue" && str === str.toLowerCase();
}
],
[
"upper",
(operand) => {
const str = operand.value;
return operand.type === "StringValue" && str === str.toUpperCase();
}
],
["none", (operand) => operand.type === "NullValue"],
["defined", (operand) => operand.type !== "UndefinedValue"],
["undefined", (operand) => operand.type === "UndefinedValue"],
["equalto", (a, b) => a.value === b.value],
["eq", (a, b) => a.value === b.value]
]);
/**
* Set the value of a variable in the current environment.
*/
set(name, value) {
return this.declareVariable(name, convertToRuntimeValues(value));
}
declareVariable(name, value) {
if (this.variables.has(name)) {
throw new SyntaxError(`Variable already declared: ${name}`);
}
this.variables.set(name, value);
return value;
}
// private assignVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
// const env = this.resolve(name);
// env.variables.set(name, value);
// return value;
// }
/**
* Set variable in the current scope.
* See https://jinja.palletsprojects.com/en/3.0.x/templates/#assignments for more information.
*/
setVariable(name, value) {
this.variables.set(name, value);
return value;
}
/**
* Resolve the environment in which the variable is declared.
* @param {string} name The name of the variable.
* @returns {Environment} The environment in which the variable is declared.
*/
resolve(name) {
if (this.variables.has(name)) {
return this;
}
if (this.parent) {
return this.parent.resolve(name);
}
throw new Error(`Unknown variable: ${name}`);
}
lookupVariable(name) {
try {
return this.resolve(name).variables.get(name) ?? new UndefinedValue();
} catch {
return new UndefinedValue();
}
}
};
function setupGlobals(env) {
env.set("false", false);
env.set("true", true);
env.set("none", null);
env.set("raise_exception", (args) => {
throw new Error(args);
});
env.set("range", range);
env.set("strftime_now", strftime_now);
env.set("True", true);
env.set("False", false);
env.set("None", null);
}
var Interpreter = class {
global;
constructor(env) {
this.global = env ?? new Environment();
}
/**
* Run the program.
*/
run(program) {
return this.evaluate(program, this.global);
}
/**
* Evaluates expressions following the binary operation type.
*/
evaluateBinaryExpression(node, environment) {
const left = this.evaluate(node.left, environment);
switch (node.operator.value) {
case "and":
return left.__bool__().value ? this.evaluate(node.right, environment) : left;
case "or":
return left.__bool__().value ? left : this.evaluate(node.right, environment);
}
const right = this.evaluate(node.right, environment);
switch (node.operator.value) {
case "==":
return new BooleanValue(left.value == right.value);
case "!=":
return new BooleanValue(left.value != right.value);
}
if (left instanceof UndefinedValue || right instanceof UndefinedValue) {
if (right instanceof UndefinedValue && ["in", "not in"].includes(node.operator.value)) {
return new BooleanValue(node.operator.value === "not in");
}
throw new Error(`Cannot perform operation ${node.operator.value} on undefined values`);
} else if (left instanceof NullValue || right instanceof NullValue) {
throw new Error("Cannot perform operation on null values");
} else if (node.operator.value === "~") {
return new StringValue(left.value.toString() + right.value.toString());
} else if ((left instanceof IntegerValue || left instanceof FloatValue) && (right instanceof IntegerValue || right instanceof FloatValue)) {
const a = left.value, b = right.value;
switch (node.operator.value) {
case "+":
case "-":
case "*": {
const res = node.operator.value === "+" ? a + b : node.operator.value === "-" ? a - b : a * b;
const isFloat = left instanceof FloatValue || right instanceof FloatValue;
return isFloat ? new FloatValue(res) : new IntegerValue(res);
}
case "/":
return new FloatValue(a / b);
case "%": {
const rem = a % b;
const isFloat = left instanceof FloatValue || right instanceof FloatValue;
return isFloat ? new FloatValue(rem) : new IntegerValue(rem);
}
case "<":
return new BooleanValue(a < b);
case ">":
return new BooleanValue(a > b);
case ">=":
return new BooleanValue(a >= b);
case "<=":
return new BooleanValue(a <= b);
}
} else if (left instanceof ArrayValue && right instanceof ArrayValue) {
switch (node.operator.value) {
case "+":