pxt-core
Version:
Microsoft MakeCode provides Blocks / JavaScript / Python tools and editors
1,505 lines (1,503 loc) • 50.2 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.convertAsync = void 0;
const nodeutil = require("./nodeutil");
const fs = require("fs");
var U = pxt.Util;
var B = pxt.blocks;
/* eslint-disable no-trailing-spaces */
const convPy = `
import ast
import sys
import json
def to_json(val):
if val is None or isinstance(val, (bool, str, int, float)):
return val
if isinstance(val, list):
return [to_json(x) for x in val]
if isinstance(val, ast.AST):
js = dict()
js['kind'] = val.__class__.__name__
for attr_name in dir(val):
if not attr_name.startswith("_"):
js[attr_name] = to_json(getattr(val, attr_name))
return js
if isinstance(val, (bytearray, bytes)):
return [x for x in val]
raise Exception("unhandled: %s (type %s)" % (val, type(val)))
js = dict()
for fn in @files@:
js[fn] = to_json(ast.parse(open(fn, "r").read()))
print(json.dumps(js))
`;
/* eslint-enable no-trailing-spaces */
const nameMap = {
"Expr": "ExprStmt",
"arg": "Arg",
"arguments": "Arguments",
"keyword": "Keyword",
"comprehension": "Comprehension",
"alias": "Alias",
"withitem": "WithItem"
};
const simpleNames = {
"Load": true, "Store": true, "Del": true, "AugLoad": true, "AugStore": true, "Param": true, "And": true,
"Or": true, "Add": true, "Sub": true, "Mult": true, "MatMult": true, "Div": true, "Mod": true, "Pow": true,
"LShift": true, "RShift": true, "BitOr": true, "BitXor": true, "BitAnd": true, "FloorDiv": true,
"Invert": true, "Not": true, "UAdd": true, "USub": true, "Eq": true, "NotEq": true, "Lt": true, "LtE": true,
"Gt": true, "GtE": true, "Is": true, "IsNot": true, "In": true, "NotIn": true,
};
function stmtTODO(v) {
pxt.tickEvent("python.todo", { kind: v.kind });
return B.mkStmt(B.mkText("TODO: " + v.kind));
}
function exprTODO(v) {
pxt.tickEvent("python.todo", { kind: v.kind });
return B.mkText(" {TODO: " + v.kind + "} ");
}
function docComment(cmt) {
if (cmt.trim().split(/\n/).length <= 1)
cmt = cmt.trim();
else
cmt = cmt + "\n";
return B.mkStmt(B.mkText("/** " + cmt + " */"));
}
let moduleAst = {};
function lookupSymbol(name) {
if (!name)
return null;
if (moduleAst[name])
return moduleAst[name];
let parts = name.split(".");
if (parts.length >= 2) {
let last = parts.length - 1;
let par = moduleAst[parts.slice(0, last).join(".")];
let ename = parts[last];
if (par) {
for (let stmt of par.body) {
if (stmt.kind == "ClassDef" || stmt.kind == "FunctionDef") {
if (stmt.name == ename)
return stmt;
}
if (stmt.kind == "Assign") {
let assignment = stmt;
if (assignment.targets.length == 1 && getName(assignment.targets[0]) == ename) {
return assignment;
}
}
}
}
}
return null;
}
let ctx;
let currIteration = 0;
let typeId = 0;
let numUnifies = 0;
function mkType(o = {}) {
let r = U.flatClone(o);
r.tid = ++typeId;
return r;
}
function currentScope() {
return ctx.currFun || ctx.currClass || ctx.currModule;
}
function defvar(n, opts) {
let scopeDef = currentScope();
let v = scopeDef.vars[n];
if (!v) {
v = scopeDef.vars[n] = { type: mkType(), name: n };
}
for (let k of Object.keys(opts)) {
v[k] = opts[k];
}
return v;
}
let tpString = mkType({ primType: "string" });
let tpNumber = mkType({ primType: "number" });
let tpBoolean = mkType({ primType: "boolean" });
let tpBuffer = mkType({ primType: "Buffer" });
let tpVoid = mkType({ primType: "void" });
function find(t) {
if (t.union) {
t.union = find(t.union);
return t.union;
}
return t;
}
function getFullName(n) {
let s = n;
let pref = "";
if (s.parent) {
pref = getFullName(s.parent);
if (!pref)
pref = "";
else
pref += ".";
}
let nn = n;
if (nn.name)
return pref + nn.name;
else
return pref + "?" + n.kind;
}
function applyTypeMap(s) {
let over = U.lookup(typeMap, s);
if (over)
return over;
for (let v of U.values(ctx.currModule.vars)) {
if (!v.isImport)
continue;
if (v.expandsTo == s)
return v.name;
if (v.isImport && U.startsWith(s, v.expandsTo + ".")) {
return v.name + s.slice(v.expandsTo.length);
}
}
return s;
}
function t2s(t) {
t = find(t);
if (t.primType)
return t.primType;
else if (t.classType)
return applyTypeMap(getFullName(t.classType));
else if (t.arrayType)
return t2s(t.arrayType) + "[]";
else
return "?" + t.tid;
}
let currErrs = "";
function error(t0, t1) {
currErrs += "types not compatible: " + t2s(t0) + " and " + t2s(t1) + "; ";
}
function typeCtor(t) {
if (t.primType)
return t.primType;
else if (t.classType)
return t.classType;
else if (t.arrayType)
return "array";
return null;
}
function isFree(t) {
return !typeCtor(find(t));
}
function canUnify(t0, t1) {
t0 = find(t0);
t1 = find(t1);
if (t0 === t1)
return true;
let c0 = typeCtor(t0);
let c1 = typeCtor(t1);
if (!c0 || !c1)
return true;
if (c0 !== c1)
return false;
if (c0 == "array") {
return canUnify(t0.arrayType, t1.arrayType);
}
return true;
}
function unifyClass(t, cd) {
t = find(t);
if (t.classType == cd)
return;
if (isFree(t)) {
t.classType = cd;
return;
}
unify(t, mkType({ classType: cd }));
}
function unify(t0, t1) {
t0 = find(t0);
t1 = find(t1);
if (t0 === t1)
return;
if (!canUnify(t0, t1)) {
error(t0, t1);
return;
}
if (typeCtor(t0) && !typeCtor(t1))
return unify(t1, t0);
numUnifies++;
t0.union = t1;
if (t0.arrayType && t1.arrayType)
unify(t0.arrayType, t1.arrayType);
}
function getClassField(ct, n, checkOnly = false, skipBases = false) {
if (!ct.fields)
ct.fields = {};
if (!ct.fields[n]) {
if (!skipBases)
for (let par = ct.baseClass; par; par = par.baseClass) {
if (par.fields && par.fields[n])
return par.fields[n];
}
if (checkOnly)
return null;
ct.fields[n] = {
inClass: ct,
name: n,
type: mkType()
};
}
return ct.fields[n];
}
function getTypeField(t, n, checkOnly = false) {
t = find(t);
let ct = t.classType;
if (ct)
return getClassField(ct, n, checkOnly);
return null;
}
function lookupVar(n) {
let s = currentScope();
while (s) {
let v = U.lookup(s.vars, n);
if (v)
return v;
// go to parent, excluding class scopes
do {
s = s.parent;
} while (s && s.kind == "ClassDef");
}
return null;
}
function getClassDef(e) {
let n = getName(e);
let v = lookupVar(n);
if (v)
return v.classdef;
let s = lookupSymbol(n);
if (s && s.kind == "ClassDef")
return s;
return null;
}
function typeOf(e) {
if (e.tsType) {
return find(e.tsType);
}
else {
e.tsType = mkType();
return e.tsType;
}
}
function isOfType(e, name) {
let t = typeOf(e);
if (t.classType && t.classType.name == name)
return true;
if (t2s(t) == name)
return true;
return false;
}
function resetCtx(m) {
ctx = {
currClass: null,
currFun: null,
currModule: m
};
}
function scope(f) {
const prevCtx = U.flatClone(ctx);
let r;
try {
r = f();
}
finally {
ctx = prevCtx;
}
return r;
}
function todoExpr(name, e) {
if (!e)
return B.mkText("");
return B.mkGroup([B.mkText("/* TODO: " + name + " "), e, B.mkText(" */")]);
}
function todoComment(name, n) {
if (n.length == 0)
return B.mkText("");
return B.mkGroup([B.mkText("/* TODO: " + name + " "), B.mkGroup(n), B.mkText(" */"), B.mkNewLine()]);
}
function doKeyword(k) {
let t = expr(k.value);
if (k.arg)
return B.mkInfix(B.mkText(k.arg), "=", t);
else
return B.mkGroup([B.mkText("**"), t]);
}
function doArgs(args, isMethod) {
U.assert(!args.kwonlyargs.length);
let nargs = args.args.slice();
if (isMethod) {
U.assert(nargs[0].arg == "self");
nargs.shift();
}
else {
U.assert(!nargs[0] || nargs[0].arg != "self");
}
let didx = args.defaults.length - nargs.length;
let lst = nargs.map(a => {
let v = defvar(a.arg, { isParam: true });
if (!a.type)
a.type = v.type;
let res = [quote(a.arg), typeAnnot(v.type)];
if (a.annotation)
res.push(todoExpr("annotation", expr(a.annotation)));
if (didx >= 0) {
res.push(B.mkText(" = "));
res.push(expr(args.defaults[didx]));
unify(a.type, typeOf(args.defaults[didx]));
}
didx++;
return B.mkGroup(res);
});
if (args.vararg)
lst.push(B.mkText("TODO *" + args.vararg.arg));
if (args.kwarg)
lst.push(B.mkText("TODO **" + args.kwarg.arg));
return B.H.mkParenthesizedExpression(B.mkCommaSep(lst));
}
const numOps = {
Sub: 1,
Div: 1,
Pow: 1,
LShift: 1,
RShift: 1,
BitOr: 1,
BitXor: 1,
BitAnd: 1,
FloorDiv: 1,
Mult: 1, // this can be also used on strings and arrays, but let's ignore that for now
};
const opMapping = {
Add: "+",
Sub: "-",
Mult: "*",
MatMult: "Math.matrixMult",
Div: "/",
Mod: "%",
Pow: "**",
LShift: "<<",
RShift: ">>",
BitOr: "|",
BitXor: "^",
BitAnd: "&",
FloorDiv: "Math.idiv",
And: "&&",
Or: "||",
Eq: "==",
NotEq: "!=",
Lt: "<",
LtE: "<=",
Gt: ">",
GtE: ">=",
Is: "===",
IsNot: "!==",
In: "py.In",
NotIn: "py.NotIn",
};
const prefixOps = {
Invert: "~",
Not: "!",
UAdd: "P+",
USub: "P-",
};
const typeMap = {
"adafruit_bus_device.i2c_device.I2CDevice": "pins.I2CDevice"
};
function stmts(ss) {
return B.mkBlock(ss.map(stmt));
}
function exprs0(ee) {
ee = ee.filter(e => !!e);
return ee.map(expr);
}
function setupScope(n) {
if (!n.vars) {
n.vars = {};
n.parent = currentScope();
}
}
function typeAnnot(t) {
let s = t2s(t);
if (s[0] == "?")
return B.mkText(": any; /** TODO: type **/");
return B.mkText(": " + t2s(t));
}
function guardedScope(v, f) {
try {
return scope(f);
}
catch (e) {
return B.mkStmt(todoComment(`conversion failed for ${v.name || v.kind}`, []));
}
}
const stmtMap = {
FunctionDef: (n) => guardedScope(n, () => {
let isMethod = !!ctx.currClass && !ctx.currFun;
if (!isMethod)
defvar(n.name, { fundef: n });
setupScope(n);
ctx.currFun = n;
if (!n.retType)
n.retType = mkType();
let prefix = "";
let funname = n.name;
let decs = n.decorator_list.filter(d => {
if (getName(d) == "property") {
prefix = "get";
return false;
}
if (d.kind == "Attribute" && d.attr == "setter" &&
d.value.kind == "Name") {
funname = d.value.id;
prefix = "set";
return false;
}
return true;
});
let nodes = [
todoComment("decorators", decs.map(expr))
];
if (isMethod) {
let fd = getClassField(ctx.currClass, funname, false, true);
if (n.body.length == 1 && n.body[0].kind == "Raise")
n.alwaysThrows = true;
if (n.name == "__init__") {
nodes.push(B.mkText("constructor"));
unifyClass(n.retType, ctx.currClass);
}
else {
if (funname == "__get__" || funname == "__set__") {
let i2cArg = "i2cDev";
let vv = n.vars[i2cArg];
if (vv) {
let i2cDevClass = lookupSymbol("adafruit_bus_device.i2c_device.I2CDevice");
if (i2cDevClass)
unifyClass(vv.type, i2cDevClass);
}
vv = n.vars["value"];
if (funname == "__set__" && vv) {
let cf = getClassField(ctx.currClass, "__get__");
if (cf.fundef)
unify(vv.type, cf.fundef.retType);
}
let nargs = n.args.args;
if (nargs[1].arg == "obj") {
// rewrite
nargs[1].arg = i2cArg;
if (nargs[nargs.length - 1].arg == "objtype") {
nargs.pop();
n.args.defaults.pop();
}
iterPy(n, e => {
if (e.kind == "Attribute") {
let a = e;
if (a.attr == "i2c_device" && getName(a.value) == "obj") {
let nm = e;
nm.kind = "Name";
nm.id = i2cArg;
delete a.attr;
delete a.value;
}
}
});
}
funname = funname.replace(/_/g, "");
}
if (!prefix) {
prefix = funname[0] == "_" ? (fd.isProtected ? "protected" : "private") : "public";
}
nodes.push(B.mkText(prefix + " "), quote(funname));
}
fd.fundef = n;
}
else {
U.assert(!prefix);
if (n.name[0] == "_")
nodes.push(B.mkText("function "), quote(funname));
else
nodes.push(B.mkText("export function "), quote(funname));
}
nodes.push(doArgs(n.args, isMethod), n.name == "__init__" ? B.mkText("") : typeAnnot(n.retType), todoComment("returns", n.returns ? [expr(n.returns)] : []));
let body = n.body.map(stmt);
if (n.name == "__init__") {
for (let f of U.values(ctx.currClass.fields)) {
if (f.initializer) {
body.push(B.mkStmt(B.mkText(`this.${quoteStr(f.name)} = `), expr(f.initializer)));
}
}
}
nodes.push(B.mkBlock(body));
return B.mkStmt(B.mkGroup(nodes));
}),
ClassDef: (n) => guardedScope(n, () => {
setupScope(n);
defvar(n.name, { classdef: n });
U.assert(!ctx.currClass);
ctx.currClass = n;
let nodes = [
todoComment("keywords", n.keywords.map(doKeyword)),
todoComment("decorators", n.decorator_list.map(expr)),
B.mkText("export class "),
quote(n.name)
];
if (n.bases.length > 0) {
nodes.push(B.mkText(" extends "));
nodes.push(B.mkCommaSep(n.bases.map(expr)));
let b = getClassDef(n.bases[0]);
if (b)
n.baseClass = b;
}
let body = stmts(n.body);
nodes.push(body);
let fieldDefs = U.values(n.fields)
.filter(f => !f.fundef && !f.isStatic && !f.isGetSet)
.map((f) => B.mkStmt(quote(f.name), typeAnnot(f.type)));
body.children = fieldDefs.concat(body.children);
return B.mkStmt(B.mkGroup(nodes));
}),
Return: (n) => {
if (n.value) {
let f = ctx.currFun;
if (f)
unify(f.retType, typeOf(n.value));
return B.mkStmt(B.mkText("return "), expr(n.value));
}
else {
return B.mkStmt(B.mkText("return"));
}
},
AugAssign: (n) => {
let op = opMapping[n.op];
if (op.length > 3)
return B.mkStmt(B.mkInfix(expr(n.target), "=", B.H.mkCall(op, [expr(n.target), expr(n.value)])));
else
return B.mkStmt(expr(n.target), B.mkText(" " + op + "= "), expr(n.value));
},
Assign: (n) => {
if (n.targets.length != 1)
return stmtTODO(n);
let pref = "";
let isConstCall = isCallTo(n.value, "const");
let nm = getName(n.targets[0]) || "";
let isUpperCase = nm && !/[a-z]/.test(nm);
if (!ctx.currClass && !ctx.currFun && nm[0] != "_")
pref = "export ";
if (nm && ctx.currClass && !ctx.currFun) {
// class fields can't be const
isConstCall = false;
let src = expr(n.value);
let fd = getClassField(ctx.currClass, nm);
let attrTp = typeOf(n.value);
let getter = getTypeField(attrTp, "__get__", true);
if (getter) {
unify(fd.type, getter.fundef.retType);
let implNm = "_" + nm;
let fdBack = getClassField(ctx.currClass, implNm);
unify(fdBack.type, attrTp);
let setter = getTypeField(attrTp, "__set__", true);
let res = [
B.mkNewLine(),
B.mkStmt(B.mkText("private "), quote(implNm), typeAnnot(attrTp))
];
if (!getter.fundef.alwaysThrows)
res.push(B.mkStmt(B.mkText(`get ${quoteStr(nm)}()`), typeAnnot(fd.type), B.mkBlock([
B.mkText(`return this.${quoteStr(implNm)}.get(this.i2c_device)`),
B.mkNewLine()
])));
if (!setter.fundef.alwaysThrows)
res.push(B.mkStmt(B.mkText(`set ${quoteStr(nm)}(value`), typeAnnot(fd.type), B.mkText(`) `), B.mkBlock([
B.mkText(`this.${quoteStr(implNm)}.set(this.i2c_device, value)`),
B.mkNewLine()
])));
fdBack.initializer = n.value;
fd.isGetSet = true;
fdBack.isGetSet = true;
return B.mkGroup(res);
}
else if (currIteration < 2) {
return B.mkText("/* skip for now */");
}
unify(fd.type, typeOf(n.targets[0]));
fd.isStatic = true;
pref = "static ";
}
unify(typeOf(n.targets[0]), typeOf(n.value));
if (isConstCall || isUpperCase) {
// first run would have "let" in it
defvar(getName(n.targets[0]), {});
let s = pref;
if (!/^static /.test(pref))
s += "const ";
return B.mkStmt(B.mkText(s), B.mkInfix(expr(n.targets[0]), "=", expr(n.value)));
}
if (!pref && n.targets[0].kind == "Tuple") {
let res = [
B.mkStmt(B.mkText("const tmp = "), expr(n.value))
];
let tup = n.targets[0];
tup.elts.forEach((e, i) => {
res.push(B.mkStmt(B.mkInfix(expr(e), "=", B.mkText("tmp[" + i + "]"))));
});
return B.mkGroup(res);
}
return B.mkStmt(B.mkText(pref), B.mkInfix(expr(n.targets[0]), "=", expr(n.value)));
},
For: (n) => {
U.assert(n.orelse.length == 0);
if (isCallTo(n.iter, "range")) {
let r = n.iter;
let def = expr(n.target);
let ref = quote(getName(n.target));
unify(typeOf(n.target), tpNumber);
let start = r.args.length == 1 ? B.mkText("0") : expr(r.args[0]);
let stop = expr(r.args[r.args.length == 1 ? 0 : 1]);
return B.mkStmt(B.mkText("for ("), B.mkInfix(def, "=", start), B.mkText("; "), B.mkInfix(ref, "<", stop), B.mkText("; "), r.args.length >= 3 ?
B.mkInfix(ref, "+=", expr(r.args[2])) :
B.mkInfix(null, "++", ref), B.mkText(")"), stmts(n.body));
}
unify(typeOf(n.iter), mkType({ arrayType: typeOf(n.target) }));
return B.mkStmt(B.mkText("for ("), expr(n.target), B.mkText(" of "), expr(n.iter), B.mkText(")"), stmts(n.body));
},
While: (n) => {
U.assert(n.orelse.length == 0);
return B.mkStmt(B.mkText("while ("), expr(n.test), B.mkText(")"), stmts(n.body));
},
If: (n) => {
let innerIf = (n) => {
let nodes = [
B.mkText("if ("),
expr(n.test),
B.mkText(")"),
stmts(n.body)
];
if (n.orelse.length) {
nodes[nodes.length - 1].noFinalNewline = true;
if (n.orelse.length == 1 && n.orelse[0].kind == "If") {
// else if
nodes.push(B.mkText(" else "));
U.pushRange(nodes, innerIf(n.orelse[0]));
}
else {
nodes.push(B.mkText(" else"), stmts(n.orelse));
}
}
return nodes;
};
return B.mkStmt(B.mkGroup(innerIf(n)));
},
With: (n) => {
if (n.items.length == 1 && isOfType(n.items[0].context_expr, "pins.I2CDevice")) {
let it = n.items[0];
let id = getName(it.optional_vars);
let res = [];
let devRef = expr(it.context_expr);
if (id) {
let v = defvar(id, { isLocal: true });
id = quoteStr(id);
res.push(B.mkStmt(B.mkText("const " + id + " = "), devRef));
unify(typeOf(it.context_expr), v.type);
devRef = B.mkText(id);
}
res.push(B.mkStmt(B.mkInfix(devRef, ".", B.mkText("begin()"))));
U.pushRange(res, n.body.map(stmt));
res.push(B.mkStmt(B.mkInfix(devRef, ".", B.mkText("end()"))));
return B.mkGroup(res);
}
let cleanup = [];
let stmts = n.items.map((it, idx) => {
let varName = "with" + idx;
if (it.optional_vars) {
let id = getName(it.optional_vars);
U.assert(id != null);
defvar(id, { isLocal: true });
varName = quoteStr(id);
}
cleanup.push(B.mkStmt(B.mkText(varName + ".end()")));
return B.mkStmt(B.mkText("const " + varName + " = "), B.mkInfix(expr(it.context_expr), ".", B.mkText("begin()")));
});
U.pushRange(stmts, n.body.map(stmt));
U.pushRange(stmts, cleanup);
return B.mkBlock(stmts);
},
Raise: (n) => {
let ex = n.exc || n.cause;
if (!ex)
return B.mkStmt(B.mkText("throw"));
let msg;
if (ex && ex.kind == "Call") {
let cex = ex;
if (cex.args.length == 1) {
msg = expr(cex.args[0]);
}
}
// didn't find string - just compile and quote; and hope for the best
if (!msg)
msg = B.mkGroup([B.mkText("`"), expr(ex), B.mkText("`")]);
return B.mkStmt(B.H.mkCall("control.fail", [msg]));
},
Assert: (n) => B.mkStmt(B.H.mkCall("control.assert", exprs0([n.test, n.msg]))),
Import: (n) => {
for (let nm of n.names) {
if (nm.asname)
defvar(nm.asname, {
expandsTo: nm.name
});
defvar(nm.name, {
isPlainImport: true
});
}
return B.mkText("");
},
ImportFrom: (n) => {
let res = [];
for (let nn of n.names) {
if (nn.name == "*")
defvar(n.module, {
isImportStar: true
});
else {
let fullname = n.module + "." + nn.name;
let sym = lookupSymbol(fullname);
let currname = nn.asname || nn.name;
if (sym && sym.kind == "Module") {
defvar(currname, {
isImport: sym,
expandsTo: fullname
});
res.push(B.mkStmt(B.mkText(`import ${quoteStr(currname)} = ${fullname}`)));
}
else {
defvar(currname, {
expandsTo: fullname
});
}
}
}
return B.mkGroup(res);
},
ExprStmt: (n) => n.value.kind == "Str" ?
docComment(n.value.s) :
B.mkStmt(expr(n.value)),
Pass: (n) => B.mkStmt(B.mkText(";")),
Break: (n) => B.mkStmt(B.mkText("break")),
Continue: (n) => B.mkStmt(B.mkText("break")),
Delete: (n) => stmtTODO(n),
Try: (n) => {
let r = [
B.mkText("try"),
stmts(n.body.concat(n.orelse)),
];
for (let e of n.handlers) {
r.push(B.mkText("catch ("), e.name ? quote(e.name) : B.mkText("_"));
// This isn't JS syntax, but PXT doesn't support try at all anyway
if (e.type)
r.push(B.mkText("/* instanceof "), expr(e.type), B.mkText(" */"));
r.push(B.mkText(")"), stmts(e.body));
}
if (n.finalbody.length)
r.push(B.mkText("finally"), stmts(n.finalbody));
return B.mkStmt(B.mkGroup(r));
},
AnnAssign: (n) => stmtTODO(n),
AsyncFunctionDef: (n) => stmtTODO(n),
AsyncFor: (n) => stmtTODO(n),
AsyncWith: (n) => stmtTODO(n),
Global: (n) => B.mkStmt(B.mkText("TODO: global: "), B.mkGroup(n.names.map(B.mkText))),
Nonlocal: (n) => B.mkStmt(B.mkText("TODO: nonlocal: "), B.mkGroup(n.names.map(B.mkText))),
};
function possibleDef(n) {
let id = n.id;
if (n.isdef === undefined) {
let curr = lookupVar(id);
if (!curr) {
if (ctx.currClass && !ctx.currFun) {
n.isdef = false; // field
curr = defvar(id, {});
}
else {
n.isdef = true;
curr = defvar(id, { isLocal: true });
}
}
else {
n.isdef = false;
}
unify(n.tsType, curr.type);
}
if (n.isdef)
return B.mkGroup([B.mkText("let "), quote(id)]);
else
return quote(id);
}
function quoteStr(id) {
if (B.isReservedWord(id))
return id + "_";
else if (!id)
return id;
else
return id;
//return id.replace(/([a-z0-9])_([a-zA-Z0-9])/g, (f: string, x: string, y: string) => x + y.toUpperCase())
}
function getName(e) {
if (e == null)
return null;
if (e.kind == "Name") {
let s = e.id;
let v = lookupVar(s);
if (v && v.expandsTo)
return v.expandsTo;
else
return s;
}
if (e.kind == "Attribute") {
let pref = getName(e.value);
if (pref)
return pref + "." + e.attr;
}
return null;
}
function quote(id) {
if (id == "self")
return B.mkText("this");
return B.mkText(quoteStr(id));
}
function isCallTo(n, fn) {
if (n.kind != "Call")
return false;
let c = n;
return getName(c.func) == fn;
}
function binop(left, pyName, right) {
let op = opMapping[pyName];
U.assert(!!op);
if (op.length > 3)
return B.H.mkCall(op, [left, right]);
else
return B.mkInfix(left, op, right);
}
let funMap = {
"memoryview": { n: "", t: tpBuffer },
"const": { n: "", t: tpNumber },
"micropython.const": { n: "", t: tpNumber },
"int": { n: "Math.trunc", t: tpNumber },
"len": { n: ".length", t: tpNumber },
"min": { n: "Math.min", t: tpNumber },
"max": { n: "Math.max", t: tpNumber },
"string.lower": { n: ".toLowerCase()", t: tpString },
"ord": { n: ".charCodeAt(0)", t: tpNumber },
"bytearray": { n: "pins.createBuffer", t: tpBuffer },
"bytes": { n: "pins.createBufferFromArray", t: tpBuffer },
"ustruct.pack": { n: "pins.packBuffer", t: tpBuffer },
"ustruct.pack_into": { n: "pins.packIntoBuffer", t: tpVoid },
"ustruct.unpack": { n: "pins.unpackBuffer", t: mkType({ arrayType: tpNumber }) },
"ustruct.unpack_from": { n: "pins.unpackBuffer", t: mkType({ arrayType: tpNumber }) },
"ustruct.calcsize": { n: "pins.packedSize", t: tpNumber },
"pins.I2CDevice.read_into": { n: ".readInto", t: tpVoid },
"bool": { n: "!!", t: tpBoolean },
"time.sleep": { n: "pause", t: tpVoid, scale: 1000 }
};
function isSuper(v) {
return isCallTo(v, "super") && v.args.length == 0;
}
function isThis(v) {
return v.kind == "Name" && v.id == "self";
}
function sourceAt(e) {
return (ctx.currModule.source[e.lineno - 1] || "").slice(e.col_offset);
}
const exprMap = {
BoolOp: (n) => {
let r = expr(n.values[0]);
for (let i = 1; i < n.values.length; ++i) {
r = binop(r, n.op, expr(n.values[i]));
}
return r;
},
BinOp: (n) => {
if (n.op == "Mod" && n.left.kind == "Str" &&
(n.right.kind == "Tuple" || n.right.kind == "List")) {
let fmt = n.left.s;
let elts = n.right.elts;
elts = elts.slice();
let res = [B.mkText("`")];
fmt.replace(/([^%]+)|(%[\d\.]*([a-zA-Z%]))/g, (f, reg, f2, flet) => {
if (reg)
res.push(B.mkText(reg.replace(/[`\\$]/g, f => "\\" + f)));
else {
let ee = elts.shift();
let et = ee ? expr(ee) : B.mkText("???");
res.push(B.mkText("${"), et, B.mkText("}"));
}
return "";
});
res.push(B.mkText("`"));
return B.mkGroup(res);
}
let r = binop(expr(n.left), n.op, expr(n.right));
if (numOps[n.op]) {
unify(typeOf(n.left), tpNumber);
unify(typeOf(n.right), tpNumber);
unify(n.tsType, tpNumber);
}
return r;
},
UnaryOp: (n) => {
let op = prefixOps[n.op];
U.assert(!!op);
return B.mkInfix(null, op, expr(n.operand));
},
Lambda: (n) => exprTODO(n),
IfExp: (n) => B.mkInfix(B.mkInfix(expr(n.test), "?", expr(n.body)), ":", expr(n.orelse)),
Dict: (n) => exprTODO(n),
Set: (n) => exprTODO(n),
ListComp: (n) => exprTODO(n),
SetComp: (n) => exprTODO(n),
DictComp: (n) => exprTODO(n),
GeneratorExp: (n) => {
if (n.generators.length == 1 && n.generators[0].kind == "Comprehension") {
let comp = n.generators[0];
if (comp.ifs.length == 0) {
return scope(() => {
let v = getName(comp.target);
defvar(v, { isParam: true }); // TODO this leaks the scope...
return B.mkInfix(expr(comp.iter), ".", B.H.mkCall("map", [
B.mkGroup([quote(v), B.mkText(" => "), expr(n.elt)])
]));
});
}
}
return exprTODO(n);
},
Await: (n) => exprTODO(n),
Yield: (n) => exprTODO(n),
YieldFrom: (n) => exprTODO(n),
Compare: (n) => {
if (n.ops.length == 1 && (n.ops[0] == "In" || n.ops[0] == "NotIn")) {
if (find(typeOf(n.comparators[0])) == tpString)
unify(typeOf(n.left), tpString);
let idx = B.mkInfix(expr(n.comparators[0]), ".", B.H.mkCall("indexOf", [expr(n.left)]));
return B.mkInfix(idx, n.ops[0] == "In" ? ">=" : "<", B.mkText("0"));
}
let r = binop(expr(n.left), n.ops[0], expr(n.comparators[0]));
for (let i = 1; i < n.ops.length; ++i) {
r = binop(r, "And", binop(expr(n.comparators[i - 1]), n.ops[i], expr(n.comparators[i])));
}
return r;
},
Call: (n) => {
let cd = getClassDef(n.func);
let recvTp;
let recv;
let methName;
let fd;
if (cd) {
if (cd.fields) {
let ff = cd.fields["__init__"];
if (ff)
fd = ff.fundef;
}
}
else {
if (n.func.kind == "Attribute") {
let attr = n.func;
recv = attr.value;
recvTp = typeOf(recv);
methName = attr.attr;
let field = getTypeField(recvTp, methName);
if (field) {
if (isSuper(recv) || (isThis(recv) && field.inClass != ctx.currClass)) {
field.isProtected = true;
}
}
if (field && field.fundef)
fd = field.fundef;
}
if (!fd) {
let name = getName(n.func);
let v = lookupVar(name);
if (v)
fd = v.fundef;
}
}
let allargs = [];
let fdargs = fd ? fd.args.args : [];
if (fdargs[0] && fdargs[0].arg == "self")
fdargs = fdargs.slice(1);
for (let i = 0; i < n.args.length; ++i) {
let e = n.args[i];
allargs.push(expr(e));
if (fdargs[i] && fdargs[i].type) {
unify(typeOf(e), fdargs[i].type);
}
}
if (fd) {
unify(typeOf(n), fd.retType);
}
let nm = getName(n.func);
let over = U.lookup(funMap, nm);
if (over) {
methName = "";
recv = null;
}
if (methName) {
nm = t2s(recvTp) + "." + methName;
over = U.lookup(funMap, nm);
if (!over && find(recvTp).arrayType) {
nm = "Array." + methName;
over = U.lookup(funMap, nm);
}
}
if (n.keywords.length > 0) {
if (fd && !fd.args.kwarg) {
let formals = fdargs.slice(n.args.length);
let defls = fd.args.defaults.slice();
while (formals.length > 0) {
let last = formals[formals.length - 1];
if (n.keywords.some(k => k.arg == last.arg))
break;
formals.pop();
defls.pop();
}
while (defls.length > formals.length)
defls.shift();
while (defls.length < formals.length)
defls.unshift(null);
let actual = U.toDictionary(n.keywords, k => k.arg);
let idx = 0;
for (let formal of formals) {
let ex = U.lookup(actual, formal.arg);
if (ex)
allargs.push(expr(ex.value));
else {
allargs.push(expr(defls[idx]));
}
idx++;
}
}
else {
let keywords = n.keywords.slice();
if (recv && isOfType(recv, "pins.I2CDevice")) {
let stopArg = null;
let startArg = null;
let endArg = null;
keywords = keywords.filter(kw => {
if (kw.arg == "stop") {
if (kw.value.kind == "NameConstant") {
let vv = kw.value.value;
if (vv === false)
stopArg = B.mkText("true");
else
stopArg = B.mkText("false");
}
else {
stopArg = B.mkInfix(null, "!", expr(kw.value));
}
return false;
}
else if (kw.arg == "start") {
startArg = expr(kw.value);
return false;
}
else if (kw.arg == "end") {
endArg = expr(kw.value);
return false;
}
return true;
});
if (endArg && !startArg)
startArg = B.mkText("0");
if (methName == "read_into") {
if (startArg) {
allargs.push(stopArg || B.mkText("false"));
allargs.push(startArg);
}
if (endArg)
allargs.push(endArg);
}
else {
if (stopArg)
allargs.push(stopArg);
if (startArg || endArg) {
allargs[0] = B.mkInfix(allargs[0], ".", B.H.mkCall("slice", endArg ? [startArg, endArg] : [startArg]));
}
}
}
if (keywords.length) {
let kwargs = keywords.map(kk => B.mkGroup([quote(kk.arg), B.mkText(": "), expr(kk.value)]));
allargs.push(B.mkGroup([
B.mkText("{"),
B.mkCommaSep(kwargs),
B.mkText("}")
]));
}
}
}
if (nm == "super" && allargs.length == 0) {
if (ctx.currClass && ctx.currClass.baseClass)
unifyClass(n.tsType, ctx.currClass.baseClass);
return B.mkText("super");
}
if (over != null) {
if (recv)
allargs.unshift(expr(recv));
let overName = over.n;
if (over.t)
unify(typeOf(n), over.t);
if (over.scale) {
allargs = allargs.map(a => {
let s = "?";
if (a.type == B.NT.Prefix && a.children.length == 0)
s = a.op;
let n = parseFloat(s);
if (!isNaN(n)) {
return B.mkText((over.scale * n) + "");
}
else {
return B.mkInfix(a, "*", B.mkText(over.scale + ""));
}
});
}
if (overName == "") {
if (allargs.length == 1)
return allargs[0];
}
else if (overName[0] == ".") {
if (allargs.length == 1)
return B.mkInfix(allargs[0], ".", B.mkText(overName.slice(1)));
else
return B.mkInfix(allargs[0], ".", B.H.mkCall(overName.slice(1), allargs.slice(1)));
}
else {
return B.H.mkCall(overName, allargs);
}
}
let fn = expr(n.func);
if (recvTp && recvTp.arrayType) {
if (methName == "append") {
methName = "push";
unify(typeOf(n.args[0]), recvTp.arrayType);
}
fn = B.mkInfix(expr(recv), ".", B.mkText(methName));
}
let nodes = [
fn,
B.mkText("("),
B.mkCommaSep(allargs),
B.mkText(")")
];
if (cd) {
nodes[0] = B.mkText(applyTypeMap(getFullName(cd)));
nodes.unshift(B.mkText("new "));
}
return B.mkGroup(nodes);
},
Num: (n) => {
unify(n.tsType, tpNumber);
let src = sourceAt(n);
let m = /^(0[box][0-9a-f]+)/i.exec(src);
if (m)
return B.mkText(m[1]);
return B.mkText(n.n + "");
},
Str: (n) => {
unify(n.tsType, tpString);
return B.mkText(B.stringLit(n.s));
},
FormattedValue: (n) => exprTODO(n),
JoinedStr: (n) => exprTODO(n),
Bytes: (n) => {
return B.mkText(`hex \`${U.toHex(new Uint8Array(n.s))}\``);
},
NameConstant: (n) => {
if (n.value != null)
unify(n.tsType, tpBoolean);
return B.mkText(JSON.stringify(n.value));
},
Ellipsis: (n) => exprTODO(n),
Constant: (n) => exprTODO(n),
Attribute: (n) => {
let part = typeOf(n.value);
let fd = getTypeField(part, n.attr);
if (fd)
unify(n.tsType, fd.type);
return B.mkInfix(expr(n.value), ".", B.mkText(quoteStr(n.attr)));
},
Subscript: (n) => {
if (n.slice.kind == "Index") {
let idx = n.slice.value;
if (currIteration > 2 && isFree(typeOf(idx))) {
unify(typeOf(idx), tpNumber);
}
return B.mkGroup([
expr(n.value),
B.mkText("["),
expr(idx),
B.mkText("]"),
]);
}
else if (n.slice.kind == "Slice") {
let s = n.slice;
return B.mkInfix(expr(n.value), ".", B.H.mkCall("slice", [s.lower ? expr(s.lower) : B.mkText("0"),
s.upper ? expr(s.upper) : null].filter(x => !!x)));
}
else {
return exprTODO(n);
}
},
Starred: (n) => B.mkGroup([B.mkText("... "), expr(n.value)]),
Name: (n) => {
if (n.id == "self" && ctx.currClass) {
unifyClass(n.tsType, ctx.currClass);
}
else {
let v = lookupVar(n.id);
if (v) {
unify(n.tsType, v.type);
if (v.isImport)
return quote(n.id); // it's import X = Y.Z.X, use X not Y.Z.X
}
}
if (n.ctx.indexOf("Load") >= 0) {
let nm = getName(n);
return quote(nm);
}
else
return possibleDef(n);
},
List: mkArrayExpr,
Tuple: mkArrayExpr,
};
function mkArrayExpr(n) {
unify(n.tsType, mkType({ arrayType: n.elts[0] ? typeOf(n.elts[0]) : mkType() }));
return B.mkGroup([
B.mkText("["),
B.mkCommaSep(n.elts.map(expr)),
B.mkText("]"),
]);
}
function expr(e) {
let f = exprMap[e.kind];
if (!f) {
pxt.tickEvent("python.todo", { kind: e.kind });
U.oops(e.kind + " - unknown expr");
}
typeOf(e);
return f(e);
}
function stmt(e) {
let f = stmtMap[e.kind];
if (!f) {
pxt.tickEvent("python.todo", { kind: e.kind });
U.oops(e.kind + " - unknown stmt");
}
let cmts = [];
let scmts = ctx.currModule.comments;
if (scmts) {
for (let i = 0; i < e.lineno; ++i) {
if (scmts[i]) {
cmts.push(scmts[i]);
scmts[i] = null;
}
}
}
let r = f(e);
if (currErrs) {
cmts.push("TODO: (below) " + currErrs);
currErrs = "";
}
if (cmts.length) {
r = B.mkGroup(cmts.map(c => B.mkStmt(B.H.mkComment(c))).concat(r));
}
return r;
}
function isEmpty(b) {
if (!b)
return true;
if (b.type == B.NT.Prefix && b.op == "")
return b.children.every(isEmpty);
if (b.type == B.NT.NewLine)
return true;
return false;
}
// TODO look at scopes of let
function toTS(mod) {
U.assert(mod.kind == "Module");
resetCtx(mod);
if (!mod.vars)
mod.vars = {};
let res = mod.body.map(stmt);
if (res.every(isEmpty))
return null;
return [
B.mkText("namespace " + mod.name + " "),
B.mkBlock(res)
];
}
function parseComments(mod) {
mod.comments = mod.source.map(l => {
let m = /(\s|^)#\s*(.*)/.exec(l);
if (m)
return m[2];
return null;
});
}
function iterPy(e, f) {
if (!e)
return;
f(e);
U.iterMap(e, (k, v) => {
if (!v || k == "parent")
return;
if (v && v.kind)
iterPy(v, f);
else if (Array.isArray(v))
v.forEach((x) => iterPy(x, f));
});
}
function parseWithPythonAsync(files) {
return nodeutil.spawnWithPipeAsync({
cmd: process.env["PYTHON3"] || (/^win/i.test(process.platform) ? "py" : "python3"),
args: [],
input: convPy.replace("@files@", JSON.stringify(files)),
silent: true
})
.then(buf => {
pxt.debug(`analyzing python AST (${buf.length} bytes)`);
let js = JSON.parse(buf.toString("utf8"));
// nodeutil.writeFileSync("pyast.json", JSON.stringify(js, null, 2), { encoding: "utf8" })
const rec = (v) => {
if (Array.isArray(v)) {
for (let i = 0; i < v.length; ++i)
v[i] = rec(v[i]);
return v;
}
if (!v || !v.kind)
return v;
v.kind = U.lookup(nameMap, v.kind) || v.kind;
if (U.lookup(simpleNames, v.kind))
return v.kind;
for (let k of Object.keys(v)) {
v[k] = rec(v[k]);
}
return v;
};
js.kind = "FileSet";
js = rec(js);
delete js.kind;
//nodeutil.writeFileSync("pyast2.json", JSON.stringify(js, null, 2), { encoding: "utf8" })
return js;
});
}
function convertAsync(fns, useInternal = false) {
let mainFiles = [];
while (/\.py$/.test(fns[0])) {
mainFiles.push(fns.shift().replace(/\\/g, "/"));
}
if (useInternal) {
return parseWithPythonAsync(mainFiles)
.then(parsedPy => {
for (let f of mainFiles) {
pxt.log(`parse: ${f}`);
let source = fs.readFileSync(f, "utf8");
let tokens = pxt.py.lex(source);
//console.log(pxt.py.tokensToString(tokens))
let res = pxt.py.parse(source, f, tokens);
let custompy = pxt.py.dump(res.stmts, true);
let realpy = pxt.py.dump(parsedPy[f].body, true);
let path = "tmp/";
if (custompy != realpy) {
fs.writeFileSync(path + "pxtpy.txt", custompy);
fs.writeFileSync(path + "realpy.txt", realpy);
fs.writeFileSync(path + "realpy.json", JSON.stringify(parsedPy[f]));
return nodeutil.spawnWithPipeAsync({
cmd: "diff",
args: ["-u", path + "pxtpy.txt", path + "realpy.txt"],
input: "",
silent: true,
allowNonZeroExit: true
})
.then(buf => {
fs.writeFileSync(path + "diff.patch", buf);
console.log(`Differences at ${f}; files written in ${path}`);
});
}
}
return Promise.resolve();
});
}
let primFiles = U.toDictionary(mainFiles.length ? mainFiles : nodeutil.allFiles(fns[0]), s => s.replace(/\\/g, "/"));
let files = U.concat(fns.map(f => nodeutil.allFiles(f))).map(f => f.replace(/\\/g, "/"));
let dirs = {};
for (let f of files) {
for (let suff of ["/docs/conf.py", "/conf.py", "/setup.py", "/README.md", "/README.rst", "/__init__.py"]) {
if (U.endsWith(f, suff)) {
const dirName = f.slice(0, f.length - suff.length);
dirs[dirName] = 1;
}
}
}
let pkgFiles = {};
for (let f of files) {
if (U.endsWith(f, ".py") && !U.endsWith(f, "/setup.py") && !U.endsWith(f, "/conf.py")) {
let par = f;
while (par) {
if (dirs[par]) {
let modName = f.slice(par.length + 1).replace(/\.py$/, "").replace(/\//g, ".");
if (!U.startsWith(modName, "examples.")) {
pkgFiles[f] = modName;
}
break;
}
par = par.replace(/\/?[^\/]*$/, "");
}
}
}
for (let m of mainFiles) {
pkgFiles[m] = m.replace(/.*\//, "").replace(/\.py$/, "");
}
const pkgFilesKeys = Object.keys(pkgFiles);
pxt.log(`files (${pkgFilesKeys.length}):\n ${pkgFilesKeys.join('\n ')}`);
return parseWithPythonAsync(pkgFilesKeys)
.then(js => {
moduleAst = {};
U.iterMap(js, (fn, js) => {
let mname = pkgFiles[fn];
js.name = mname;
js.source = fs.readFileSync(fn, "utf8").split(/\r?\n/);
moduleAst[mname] = js;
});
for (let i = 0; i < 5; ++i) {
currIteration = i;
U.iterMap(js, (fn, js) => {
pxt.log(`converting ${fn}`);
try {
toTS(js);
}
catch (e) {
console.log(e);
}
});
}
let files = {};
currIteration = 1000;
U.iterMap(js, (fn, js) => {
if (primFiles[fn]) {
pxt.debug(`converting ${fn}`);
let s = "//\n// *** " + fn + " ***\n//\n\n";
parseComments(js);
let nodes = toTS(js);
if (!nodes)
return;
let res = B.flattenNode(nodes);
s += res.output;
let fn2 = js.name.replace(/\..*/, "") + ".ts";
files[fn2] = (files[fn2] || "") + s;
}
else {
pxt.debug(`skipping ${fn}`);
}
});
U