UNPKG

pxt-core

Version:

Microsoft MakeCode provides Blocks / JavaScript / Python tools and editors

779 lines (778 loc) • 27.3 kB
"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.stringify = exports.lazyRequire = exports.lazyDependencies = exports.getBundledPackagesDocs = exports.resolveMd = exports.lastResolveMdDirs = exports.fileExistsSync = exports.openUrl = exports.writeFileSync = exports.existsDirSync = exports.allFiles = exports.cp = exports.cpR = exports.mkdirP = exports.pathToPtr = exports.getPxtTarget = exports.readPkgConfig = exports.readText = exports.readJson = exports.sanitizePath = exports.timestamp = exports.isBranchProtectedAsync = exports.createPullRequestAsync = exports.gitPushAsync = exports.npmVersionBumpAsync = exports.getLocalTagPointingAtHeadAsync = exports.switchBranchAsync = exports.createBranchAsync = exports.getGitHubOwnerAndRepoAsync = exports.getGitHubUserAsync = exports.getGitHubTokenAsync = exports.getCurrentBranchNameAsync = exports.getDefaultBranchAsync = exports.needsGitCleanAsync = exports.currGitTagAsync = exports.gitInfoAsync = exports.runGitAsync = exports.runNpmAsyncWithCwd = exports.npmRegistryAsync = exports.runNpmAsync = exports.addCmd = exports.spawnWithPipeAsync = exports.spawnAsync = exports.readResAsync = exports.setTargetDir = exports.runCliFinalizersAsync = exports.addCliFinalizer = exports.cliFinalizers = exports.pxtCoreDir = exports.targetDir = void 0; exports.matchesAny = void 0; const child_process = require("child_process"); const fs = require("fs"); const zlib = require("zlib"); const url = require("url"); const http = require("http"); const https = require("https"); const crypto = require("crypto"); const path = require("path"); const os = require("os"); var Util = pxt.Util; //This should be correct at startup when running from command line exports.targetDir = process.cwd(); exports.pxtCoreDir = path.join(__dirname, ".."); exports.cliFinalizers = []; function addCliFinalizer(f) { exports.cliFinalizers.push(f); } exports.addCliFinalizer = addCliFinalizer; function runCliFinalizersAsync() { let fins = exports.cliFinalizers; exports.cliFinalizers = []; return pxt.Util.promiseMapAllSeries(fins, f => f()) .then(() => { }); } exports.runCliFinalizersAsync = runCliFinalizersAsync; function setTargetDir(dir) { exports.targetDir = dir; module.paths.push(path.join(exports.targetDir, "node_modules")); } exports.setTargetDir = setTargetDir; function readResAsync(g) { return new Promise((resolve, reject) => { let bufs = []; g.on('data', (c) => { if (typeof c === "string") bufs.push(Buffer.from(c, "utf8")); else bufs.push(c); }); g.on("error", (err) => reject(err)); g.on('end', () => resolve(Buffer.concat(bufs))); }); } exports.readResAsync = readResAsync; function spawnAsync(opts) { opts.pipe = false; return spawnWithPipeAsync(opts) .then(() => { }); } exports.spawnAsync = spawnAsync; function spawnWithPipeAsync(opts) { // https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2 if (os.platform() === "win32" && typeof opts.shell === "undefined") opts.shell = true; if (opts.pipe === undefined) opts.pipe = true; let info = opts.cmd + " " + opts.args.join(" "); if (opts.cwd && opts.cwd != ".") info = "cd " + opts.cwd + "; " + info; //console.log("[run] " + info) // uncomment for debugging, but it can potentially leak secrets so do not check in return new Promise((resolve, reject) => { let ch = child_process.spawn(opts.cmd, opts.args, { cwd: opts.cwd, env: opts.envOverrides ? extendEnv(process.env, opts.envOverrides) : process.env, stdio: opts.pipe ? [opts.input == null ? process.stdin : "pipe", "pipe", process.stderr] : "inherit", shell: opts.shell || false }); let bufs = []; if (opts.pipe) ch.stdout.on('data', (buf) => { bufs.push(buf); if (!opts.silent) { process.stdout.write(buf); } }); ch.on('close', (code) => { if (code != 0 && !opts.allowNonZeroExit) reject(new Error("Exit code: " + code + " from " + info)); resolve(Buffer.concat(bufs)); }); if (opts.input != null) ch.stdin.end(opts.input, "utf8"); }); } exports.spawnWithPipeAsync = spawnWithPipeAsync; function extendEnv(base, overrides) { let res = {}; Object.keys(base).forEach(key => res[key] = base[key]); Object.keys(overrides).forEach(key => res[key] = overrides[key]); return res; } function addCmd(name) { return name + (/^win/.test(process.platform) ? ".cmd" : ""); } exports.addCmd = addCmd; function runNpmAsync(...args) { return runNpmAsyncWithCwd(".", ...args); } exports.runNpmAsync = runNpmAsync; function npmRegistryAsync(pkg) { // TODO: use token if available return Util.httpGetJsonAsync(`https://registry.npmjs.org/${pkg}`); } exports.npmRegistryAsync = npmRegistryAsync; function runNpmAsyncWithCwd(cwd, ...args) { return spawnAsync({ cmd: addCmd("npm"), args: args, cwd }); } exports.runNpmAsyncWithCwd = runNpmAsyncWithCwd; function runGitAsync(...args) { return spawnAsync({ cmd: "git", args: args, cwd: "." }); } exports.runGitAsync = runGitAsync; function gitInfoAsync(args, cwd, silent = false) { return Promise.resolve() .then(() => spawnWithPipeAsync({ cmd: "git", args: args, cwd, silent })) .then(buf => buf.toString("utf8").trim()); } exports.gitInfoAsync = gitInfoAsync; function currGitTagAsync() { return gitInfoAsync(["describe", "--tags", "--exact-match"]) .then(t => { if (!t) Util.userError("no git tag found"); return t; }); } exports.currGitTagAsync = currGitTagAsync; function needsGitCleanAsync() { return Promise.resolve() .then(() => spawnWithPipeAsync({ cmd: "git", args: ["status", "--porcelain", "--untracked-files=no"] })) .then(buf => { if (buf.length) Util.userError("Please commit all files to git before running 'pxt bump'"); }); } exports.needsGitCleanAsync = needsGitCleanAsync; async function getDefaultBranchAsync() { const b = await gitInfoAsync(["symbolic-ref", "--short", "refs/remotes/origin/HEAD"], undefined, true); if (!b) Util.userError("no git remote branch found"); return b.replace(/^origin\//, ""); } exports.getDefaultBranchAsync = getDefaultBranchAsync; async function getCurrentBranchNameAsync() { const b = await gitInfoAsync(["rev-parse", "--abbrev-ref", "HEAD"], undefined, true); if (!b) Util.userError("no git local branch found"); return b; } exports.getCurrentBranchNameAsync = getCurrentBranchNameAsync; async function getGitHubTokenAsync() { const outputBuf = await spawnWithPipeAsync({ cmd: "git", args: ["credential", "fill"], input: "protocol=https\nhost=github.com\n\n", silent: true }); const output = outputBuf.toString("utf8").trim(); const lines = output.split("\n"); const creds = {}; for (const line of lines) { const [key, ...rest] = line.split("="); creds[key] = rest.join("="); } if (creds.password) { return creds.password; } else { Util.userError("No GitHub credentials found via git credential helper."); } } exports.getGitHubTokenAsync = getGitHubTokenAsync; async function getGitHubUserAsync(token) { const res = await Util.httpRequestCoreAsync({ url: "https://api.github.com/user", method: "GET", headers: { Authorization: `token ${token}`, } }); if (res.statusCode !== 200) { Util.userError(`Failed to get GitHub username: ${res.statusCode} ${res.text}`); } const data = await res.json; return data.login; } exports.getGitHubUserAsync = getGitHubUserAsync; async function getGitHubOwnerAndRepoAsync() { const remoteUrl = await gitInfoAsync(["config", "--get", "remote.origin.url"], undefined, true); if (!remoteUrl) { Util.userError("No remote origin URL found"); } const match = remoteUrl.match(/github\.com[:\/](.+?)\/(.+?)(\.git)?$/); if (!match) { Util.userError("Invalid remote origin URL: " + remoteUrl); } const owner = match[1]; const repo = match[2]; return { owner, repo }; } exports.getGitHubOwnerAndRepoAsync = getGitHubOwnerAndRepoAsync; async function createBranchAsync(branchName) { await spawnAsync({ cmd: "git", args: ["checkout", "-b", branchName], silent: true, }); await spawnAsync({ cmd: "git", args: ["push", "--set-upstream", "origin", branchName], silent: true, }); } exports.createBranchAsync = createBranchAsync; async function switchBranchAsync(branchName) { await spawnAsync({ cmd: "git", args: ["checkout", branchName], silent: true, }); } exports.switchBranchAsync = switchBranchAsync; async function getLocalTagPointingAtHeadAsync() { try { const output = await spawnWithPipeAsync({ cmd: "git", args: ["tag", "--points-at", "HEAD"], silent: true, }); const result = output.toString("utf-8").trim(); const tags = result.split("\n").map(t => t.trim()).filter(Boolean); const versionTag = tags.find(t => /^v\d+\.\d+\.\d+$/.test(t)); return versionTag; } catch (e) { return undefined; } } exports.getLocalTagPointingAtHeadAsync = getLocalTagPointingAtHeadAsync; async function npmVersionBumpAsync(bumpType, tagCommit = true) { const output = await spawnWithPipeAsync({ cmd: addCmd("npm"), args: ["version", bumpType, "--message", quoteIfNeeded(`[pxt-cli] bump version to %s`), "--git-tag-version", tagCommit ? "true" : "false"], cwd: ".", silent: true, }); const ver = output.toString("utf8").trim(); // If not tagging, the `npm version` command will not commit the change to package.json, so we need to do it manually if (!tagCommit) { await spawnAsync({ cmd: "git", args: ["add", "package.json"], cwd: ".", silent: true, }); await spawnAsync({ cmd: "git", args: ["commit", "-m", quoteIfNeeded(`[pxt-cli] bump version to ${ver}`)], cwd: ".", silent: true, }); } return ver; } exports.npmVersionBumpAsync = npmVersionBumpAsync; function quoteIfNeeded(arg) { if (os.platform() === "win32") { return `"${arg}"`; } return arg; } function gitPushAsync(followTags = true) { const args = ["push"]; if (followTags) args.push("--follow-tags"); args.push("origin", "HEAD"); return spawnAsync({ cmd: "git", args, cwd: ".", silent: true, }); } exports.gitPushAsync = gitPushAsync; async function createPullRequestAsync(opts) { const { token, owner, repo, title, head, base, body } = opts; const res = await Util.httpRequestCoreAsync({ url: `https://api.github.com/repos/${owner}/${repo}/pulls`, method: "POST", headers: { Authorization: `token ${token}`, "Accept": "application/vnd.github+json", "Content-Type": "application/json", }, data: { title, head, base, body, }, }); if (res.statusCode !== 201) { Util.userError(`Failed to create pull request: ${res.statusCode} ${res.text}`); } const data = await res.json; return data.html_url; } exports.createPullRequestAsync = createPullRequestAsync; async function isBranchProtectedAsync(token, owner, repo, branch) { const res = await Util.httpRequestCoreAsync({ url: `https://api.github.com/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`, method: "GET", headers: { Authorization: `token ${token}`, "Accept": "application/vnd.github+json" } }); if (res.statusCode !== 200) { Util.userError(`Failed to get branch protection info: ${res.statusCode} ${res.text}`); } const data = await res.json; const requiresPR = data.protected; return requiresPR; } exports.isBranchProtectedAsync = isBranchProtectedAsync; function timestamp(date = new Date()) { const yyyy = date.getUTCFullYear(); const mm = String(date.getUTCMonth() + 1).padStart(2, "0"); const dd = String(date.getUTCDate()).padStart(2, "0"); const hh = String(date.getUTCHours()).padStart(2, "0"); const min = String(date.getUTCMinutes()).padStart(2, "0"); const sec = String(date.getUTCSeconds()).padStart(2, "0"); return `${yyyy}${mm}${dd}-${hh}${min}${sec}`; } exports.timestamp = timestamp; function nodeHttpRequestAsync(options) { let isHttps = false; let u = url.parse(options.url); if (u.protocol == "https:") isHttps = true; else if (u.protocol == "http:") isHttps = false; else return Promise.reject("bad protocol: " + u.protocol); u.headers = Util.clone(options.headers) || {}; let data = options.data; u.method = options.method || (data == null ? "GET" : "POST"); let buf = null; u.headers["accept-encoding"] = "gzip"; u.headers["user-agent"] = "PXT-CLI"; let gzipContent = false; if (data != null) { if (Buffer.isBuffer(data)) { buf = data; } else if (typeof data == "object") { buf = Buffer.from(JSON.stringify(data), "utf8"); u.headers["content-type"] = "application/json; charset=utf8"; if (options.allowGzipPost) gzipContent = true; } else if (typeof data == "string") { buf = Buffer.from(data, "utf8"); if (options.allowGzipPost) gzipContent = true; } else { Util.oops("bad data"); } } if (gzipContent) { buf = zlib.gzipSync(buf); u.headers['content-encoding'] = "gzip"; } if (buf) u.headers['content-length'] = buf.length; return new Promise((resolve, reject) => { const handleResponse = (res) => { let g = res; if (/gzip/.test(res.headers['content-encoding'])) { let tmp = zlib.createUnzip(); res.pipe(tmp); g = tmp; } resolve(readResAsync(g).then(buf => { let text = null; let json = null; try { text = buf.toString("utf8"); json = JSON.parse(text); } catch (e) { } let resp = { statusCode: res.statusCode, headers: res.headers, buffer: buf, text: text, json: json, }; return resp; })); }; const req = isHttps ? https.request(u, handleResponse) : http.request(u, handleResponse); req.on('error', (err) => reject(err)); req.end(buf); }); } function sha256(hashData) { let sha; let hash = crypto.createHash("sha256"); hash.update(hashData, "utf8"); sha = hash.digest().toString("hex").toLowerCase(); return sha; } function init() { require("promise.prototype.finally").shim(); // Make unhandled async rejections throw process.on('unhandledRejection', e => { throw e; }); Util.isNodeJS = true; Util.httpRequestCoreAsync = nodeHttpRequestAsync; Util.sha256 = sha256; Util.cpuUs = () => { const p = process.cpuUsage(); return p.system + p.user; }; Util.getRandomBuf = buf => { let tmp = crypto.randomBytes(buf.length); for (let i = 0; i < buf.length; ++i) buf[i] = tmp[i]; }; global.btoa = (str) => Buffer.from(str, "binary").toString("base64"); global.atob = (str) => Buffer.from(str, "base64").toString("binary"); } function sanitizePath(path) { return path.replace(/[^\w@\/]/g, "-").replace(/^\/+/, ""); } exports.sanitizePath = sanitizePath; function readJson(fn) { return JSON.parse(fs.readFileSync(fn, "utf8")); } exports.readJson = readJson; function readText(fn) { return fs.readFileSync(fn, "utf8"); } exports.readText = readText; function readPkgConfig(dir) { //pxt.debug("readPkgConfig in " + dir) const fn = path.join(dir, pxt.CONFIG_NAME); const js = readJson(fn); const ap = js.additionalFilePath; if (ap) { let adddir = path.join(dir, ap); if (!existsDirSync(adddir)) pxt.U.userError(`additional pxt.json not found: ${adddir} in ${dir} + ${ap}`); pxt.debug("additional pxt.json: " + adddir); const js2 = readPkgConfig(adddir); for (let k of Object.keys(js2)) { if (!js.hasOwnProperty(k)) { js[k] = js2[k]; } } js.additionalFilePaths = [ap].concat(js2.additionalFilePaths.map(d => path.join(ap, d))); } else { js.additionalFilePaths = []; } // don't inject version number // as they get serialized later on // if (!js.targetVersions) js.targetVersions = pxt.appTarget.versions; return js; } exports.readPkgConfig = readPkgConfig; function getPxtTarget() { if (fs.existsSync(exports.targetDir + "/built/target.json")) { let res = readJson(exports.targetDir + "/built/target.json"); if (res.id && res.bundledpkgs) return res; } let raw = readJson(exports.targetDir + "/pxtarget.json"); raw.bundledpkgs = {}; return raw; } exports.getPxtTarget = getPxtTarget; function pathToPtr(path) { return "ptr-" + sanitizePath(path.replace(/^ptr-/, "")).replace(/[^\w@]/g, "-"); } exports.pathToPtr = pathToPtr; function mkdirP(thePath) { if (thePath == "." || !thePath) return; if (!fs.existsSync(thePath)) { mkdirP(path.dirname(thePath)); fs.mkdirSync(thePath); } } exports.mkdirP = mkdirP; function cpR(src, dst, maxDepth = 8) { src = path.resolve(src); let files = allFiles(src, { maxDepth }); let dirs = {}; for (let f of files) { let bn = f.slice(src.length); let dd = path.join(dst, bn); let dir = path.dirname(dd); if (!Util.lookup(dirs, dir)) { mkdirP(dir); dirs[dir] = true; } let buf = fs.readFileSync(f); fs.writeFileSync(dd, buf); } } exports.cpR = cpR; function cp(srcFile, destDirectory, destName) { mkdirP(destDirectory); let dest = path.resolve(destDirectory, destName || path.basename(srcFile)); let buf = fs.readFileSync(path.resolve(srcFile)); fs.writeFileSync(dest, buf); } exports.cp = cp; function allFiles(top, opts = {}) { const { maxDepth, allowMissing, includeDirs, ignoredFileMarker, includeHiddenFiles } = Object.assign({ maxDepth: 8 }, opts); let res = []; if (allowMissing && !existsDirSync(top)) return res; for (const p of fs.readdirSync(top)) { if (p[0] == "." && !includeHiddenFiles) continue; const inner = path.join(top, p); const st = fs.statSync(inner); if (st.isDirectory()) { // check for ingored folder marker if (ignoredFileMarker && fs.existsSync(path.join(inner, ignoredFileMarker))) continue; if (maxDepth > 1) Util.pushRange(res, allFiles(inner, Object.assign(Object.assign({}, opts), { maxDepth: maxDepth - 1 }))); if (includeDirs) res.push(inner); } else { res.push(inner); } } return res; } exports.allFiles = allFiles; function existsDirSync(name) { try { const stats = fs.lstatSync(name); return stats && stats.isDirectory(); } catch (e) { return false; } } exports.existsDirSync = existsDirSync; function writeFileSync(p, data, options) { mkdirP(path.dirname(p)); fs.writeFileSync(p, data, options); if (pxt.options.debug) { const stats = fs.statSync(p); pxt.log(` + ${p} ${stats.size > 1000000 ? (stats.size / 1000000).toFixed(2) + ' m' : stats.size > 1000 ? (stats.size / 1000).toFixed(2) + 'k' : stats.size}b`); } } exports.writeFileSync = writeFileSync; function openUrl(startUrl, browser) { if (!/^[a-z0-9A-Z#=\.\-\\\/%:\?_&]+$/.test(startUrl)) { console.error("invalid URL to open: " + startUrl); return; } let cmds = { darwin: "open", win32: "start", linux: "xdg-open" }; if (/^win/.test(os.platform()) && !/^[a-z0-9]+:\/\//i.test(startUrl)) startUrl = startUrl.replace('/', '\\'); else startUrl = startUrl.replace('\\', '/'); console.log(`opening ${startUrl}`); if (browser) { child_process.spawn(getBrowserLocation(browser), [startUrl], { detached: true }); } else { child_process.exec(`${cmds[process.platform]} ${startUrl}`); } } exports.openUrl = openUrl; function getBrowserLocation(browser) { let browserPath; const normalizedBrowser = browser.toLowerCase(); if (normalizedBrowser === "chrome") { switch (os.platform()) { case "win32": browserPath = "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"; break; case "darwin": browserPath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; break; case "linux": browserPath = "/opt/google/chrome/chrome"; break; default: break; } } else if (normalizedBrowser === "firefox") { browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe"; switch (os.platform()) { case "win32": browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe"; break; case "darwin": browserPath = "/Applications/Firefox.app"; break; case "linux": default: break; } } else if (normalizedBrowser === "ie") { browserPath = "C:/Program Files/Internet Explorer/iexplore.exe"; } else if (normalizedBrowser === "safari") { browserPath = "/Applications/Safari.app/Contents/MacOS/Safari"; } if (browserPath && fs.existsSync(browserPath)) { return browserPath; } return browser; } function fileExistsSync(p) { try { let stats = fs.lstatSync(p); return stats && stats.isFile(); } catch (e) { return false; } } exports.fileExistsSync = fileExistsSync; exports.lastResolveMdDirs = []; // returns undefined if not found function resolveMd(root, pathname, md) { const docs = path.join(root, "docs"); const tryRead = (fn) => { if (fileExistsSync(fn + ".md")) return fs.readFileSync(fn + ".md", "utf8"); if (fileExistsSync(fn + "/index.md")) return fs.readFileSync(fn + "/index.md", "utf8"); return null; }; const targetMd = md ? md : tryRead(path.join(docs, pathname)); if (targetMd && !/^\s*#+\s+@extends/m.test(targetMd)) return targetMd; const dirs = [ path.join(root, "/node_modules/pxt-core/common-docs"), ...getBundledPackagesDocs() ]; for (const d of dirs) { const template = tryRead(path.join(d, pathname)); if (template) return pxt.docs.augmentDocs(template, targetMd); } return undefined; } exports.resolveMd = resolveMd; function getBundledPackagesDocs() { const handledDirectories = {}; const outputDocFolders = []; for (const bundledDir of pxt.appTarget.bundleddirs || []) { getPackageDocs(bundledDir, outputDocFolders, handledDirectories); } return outputDocFolders; /** * This needs to produce a topologically sorted array of the docs of `dir` and any required packages, * such that any package listed as a dependency / additionalFilePath of another * package is added to `folders` before the one that requires it. */ function getPackageDocs(packageDir, folders, resolvedDirs) { if (resolvedDirs[packageDir]) return; resolvedDirs[packageDir] = true; const jsonDir = path.join(packageDir, "pxt.json"); const pxtjson = fs.existsSync(jsonDir) && readJson(jsonDir); // before adding this package, include the docs of any package this one depends upon. if (pxtjson) { /** * include the package this extends from first; * that may have dependencies that overlap with this one or that will later be * overwritten by this one **/ if (pxtjson.additionalFilePath) { getPackageDocs(path.join(packageDir, pxtjson.additionalFilePath), folders, resolvedDirs); } if (pxtjson.dependencies) { Object.keys(pxtjson.dependencies).forEach(dep => { const parts = /^file:(.+)$/i.exec(pxtjson.dependencies[dep]); if (parts) { getPackageDocs(path.join(packageDir, parts[1]), folders, resolvedDirs); } }); } } const docsDir = path.join(packageDir, "docs"); if (fs.existsSync(docsDir)) { folders.push(docsDir); } } } exports.getBundledPackagesDocs = getBundledPackagesDocs; function lazyDependencies() { // find pxt-core package const deps = {}; [path.join("node_modules", "pxt-core", "package.json"), "package.json"] .filter(f => fs.existsSync(f)) .map(f => readJson(f)) .forEach(config => config && config.lazyDependencies && Util.jsonMergeFrom(deps, config.lazyDependencies)); return deps; } exports.lazyDependencies = lazyDependencies; function lazyRequire(name, install = false) { let r; try { r = require(name); } catch (e) { pxt.debug(e); pxt.debug(require.resolve.paths(name)); r = undefined; } if (!r && install) pxt.log(`package "${name}" failed to load, run "pxt npminstallnative" to install native depencencies`); return r; } exports.lazyRequire = lazyRequire; function stringify(content) { if (process.env["PXT_ENV"] === "production") { return JSON.stringify(content); } return JSON.stringify(content, null, 4); } exports.stringify = stringify; function matchesAny(input, patterns) { return patterns.some(pattern => { if (typeof pattern === "string") { return input === pattern; } else { return pattern.test(input); } }); } exports.matchesAny = matchesAny; init();