pxt-core
Version:
Microsoft MakeCode provides Blocks / JavaScript / Python tools and editors
1,242 lines • 290 kB
JavaScript
"use strict";
/* eslint-disable @typescript-eslint/no-for-in-array */
Object.defineProperty(exports, "__esModule", { value: true });
exports.mainCli = exports.getCodeSnippets = exports.getSnippets = exports.hex2uf2Async = exports.hexdumpAsync = exports.extractAsync = exports.testAsync = exports.runAsync = exports.deployAsync = exports.consoleAsync = exports.gendocsAsync = exports.buildCoreDeclarationFiles = exports.buildShareSimJsAsync = exports.buildAsync = exports.buildJResAsync = exports.buildJResSpritesAsync = exports.validateTranslatedBlocks = exports.npmInstallNativeAsync = exports.cleanGenAsync = exports.cleanAsync = exports.staticpkgAsync = exports.formatAsync = exports.downloadDiscourseTagAsync = exports.validateAndFixPkgConfig = exports.downloadPlaylistsAsync = exports.exportCppAsync = exports.timeAsync = exports.augmnetDocsAsync = exports.serviceAsync = exports.initAsync = exports.addAsync = exports.installAsync = exports.serveAsync = exports.internalBuildTargetAsync = exports.buildTargetAsync = exports.ghpPushAsync = exports.uploadTargetRefsAsync = exports.uploadTargetReleaseAsync = exports.yesNoAsync = exports.queryAsync = exports.apiAsync = exports.pokeRepoAsync = exports.globalConfig = void 0;
/// <reference path="../built/pxtlib.d.ts"/>
/// <reference path="../built/pxtcompiler.d.ts"/>
/// <reference path="../built/pxtpy.d.ts"/>
/// <reference path="../built/pxtsim.d.ts"/>
global.pxt = pxt;
const nodeutil = require("./nodeutil");
const crypto = require("crypto");
const fs = require("fs");
const os = require("os");
const path = require("path");
const child_process = require("child_process");
const util_1 = require("util");
const chalk = require('chalk');
var U = pxt.Util;
var Cloud = pxt.Cloud;
const server = require("./server");
const build = require("./buildengine");
const commandParser = require("./commandparser");
const hid = require("./hid");
const gdb = require("./gdb");
const clidbg = require("./clidbg");
const pyconv = require("./pyconv");
const gitfs = require("./gitfs");
const crowdin = require("./crowdin");
const youtube = require("./youtube");
const subwebapp_1 = require("./subwebapp");
const rimraf = require('rimraf');
pxt.docs.requireDOMSanitizer = () => require("sanitize-html");
let forceCloudBuild = process.env["KS_FORCE_CLOUD"] !== "no";
let forceLocalBuild = !!process.env["PXT_FORCE_LOCAL"];
let forceBuild = false; // don't use cache
let useCompileServiceDocker = false;
Error.stackTraceLimit = 100;
function parseHwVariant(parsed) {
let hwvariant = parsed && parsed.flags["hwvariant"];
if (hwvariant) {
// map known variants
const knowVariants = {
"f4": "stm32f401",
"f401": "stm32f401",
"d5": "samd51",
"d51": "samd51",
"p0": "rpi",
"pi0": "rpi",
"pi": "rpi"
};
hwvariant = knowVariants[hwvariant.toLowerCase()] || hwvariant;
if (!/^hw---/.test(hwvariant))
hwvariant = 'hw---' + hwvariant;
}
return hwvariant;
}
function parseBuildInfo(parsed) {
const cloud = parsed && parsed.flags["cloudbuild"];
const local = parsed && parsed.flags["localbuild"];
const useCompService = parsed === null || parsed === void 0 ? void 0 : parsed.flags["localcompileservice"];
const hwvariant = parseHwVariant(parsed);
forceBuild = parsed && !!parsed.flags["force"];
if (cloud && local)
U.userError("cannot specify local-build and cloud-build together");
if (cloud) {
forceCloudBuild = true;
forceLocalBuild = false;
}
if (local) {
forceCloudBuild = false;
forceLocalBuild = true;
}
if (useCompService) {
forceCloudBuild = false;
forceLocalBuild = true;
useCompileServiceDocker = true;
}
if (hwvariant) {
pxt.log(`setting hardware variant to ${hwvariant}`);
pxt.setHwVariant(hwvariant);
}
}
const p = new commandParser.CommandParser();
function initTargetCommands() {
let cmdsjs = path.join(nodeutil.targetDir, 'built/cmds.js');
if (fs.existsSync(cmdsjs)) {
pxt.debug(`loading cli extensions...`);
let cli = require.main.require(cmdsjs);
if (cli.deployAsync) {
pxt.commands.deployFallbackAsync = cli.deployAsync;
}
if (cli.addCommands) {
cli.addCommands(p);
}
}
}
let prevExports = global.savedModuleExports;
if (prevExports) {
module.exports = prevExports;
}
let reportDiagnostic = reportDiagnosticSimply;
const targetJsPrefix = "var pxtTargetBundle = ";
function reportDiagnostics(diagnostics) {
for (const diagnostic of diagnostics) {
reportDiagnostic(diagnostic);
}
}
function reportDiagnosticSimply(diagnostic) {
let output = pxtc.getDiagnosticString(diagnostic);
pxt.log(output);
}
function fatal(msg) {
pxt.log("Fatal error: " + msg);
throw new Error(msg);
}
exports.globalConfig = {};
function homePxtDir() {
return path.join(process.env["HOME"] || process.env["UserProfile"], ".pxt");
}
function cacheDir() {
return path.join(homePxtDir(), "cache");
}
function configPath() {
return path.join(homePxtDir(), "config.json");
}
let homeDirsMade = false;
function mkHomeDirs() {
if (homeDirsMade)
return;
homeDirsMade = true;
if (!fs.existsSync(homePxtDir()))
fs.mkdirSync(homePxtDir());
if (!fs.existsSync(cacheDir()))
fs.mkdirSync(cacheDir());
}
function saveConfig() {
mkHomeDirs();
nodeutil.writeFileSync(configPath(), JSON.stringify(exports.globalConfig, null, 4) + "\n");
}
function initConfigAsync() {
let p = Promise.resolve();
let atok = process.env["PXT_ACCESS_TOKEN"];
if (fs.existsSync(configPath())) {
let config = readJson(configPath());
exports.globalConfig = config;
}
p.then(() => {
if (atok) {
let mm = /^(https?:.*)\?access_token=([\w\-\.]+)/.exec(atok);
if (!mm) {
console.error("Invalid accessToken format, expecting something like 'https://example.com/?access_token=0.abcd.XXXX'");
return;
}
Cloud.apiRoot = mm[1].replace(/\/$/, "").replace(/\/api$/, "") + "/api/";
Cloud.accessToken = mm[2];
}
});
return p;
}
function loadGithubToken() {
if (!pxt.github.token)
pxt.github.token = process.env["GITHUB_ACCESS_TOKEN"] || process.env["GITHUB_TOKEN"];
pxt.github.db = new FileGithubDb(pxt.github.db);
}
/**
* Caches github releases under pxt_modules/.githubcache/...
*/
class FileGithubDb {
constructor(db) {
this.db = db;
}
loadAsync(repopath, tag, suffix, loader) {
// only cache releases
if (!/^v\d+\.\d+\.\d+$/.test(tag))
return loader(repopath, tag);
const dir = path.join(`pxt_modules`, `.githubcache`, ...repopath.split(/\//g), tag);
const fn = path.join(dir, suffix + ".json");
// cache hit
if (fs.existsSync(fn)) {
const json = fs.readFileSync(fn, "utf8");
const p = pxt.Util.jsonTryParse(json);
if (p) {
pxt.debug(`cache hit ${fn}`);
return Promise.resolve(p);
}
}
// download and cache
return loader(repopath, tag)
.then(p => {
if (p) {
pxt.debug(`cached ${fn}`);
nodeutil.mkdirP(dir);
fs.writeFileSync(fn, JSON.stringify(p), "utf8");
}
return p;
});
}
latestVersionAsync(repopath, config) {
return this.db.latestVersionAsync(repopath, config);
}
loadConfigAsync(repopath, tag) {
return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t));
}
loadPackageAsync(repopath, tag) {
return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
}
loadTutorialMarkdown(repopath, tag) {
return this.loadAsync(repopath, tag, "tutorial", (r, t) => this.db.loadTutorialMarkdown(r, t));
}
cacheReposAsync(resp) {
return this.db.cacheReposAsync(resp);
}
}
function searchAsync(...query) {
return pxt.packagesConfigAsync()
.then(config => pxt.github.searchAsync(query.join(" "), config))
.then(res => {
for (let r of res) {
console.log(`${r.fullName}: ${r.description}`);
}
});
}
function pokeRepoAsync(parsed) {
const repo = parsed.args[0];
let data = {
repo: repo,
getkey: false
};
if (parsed.flags["u"])
data.getkey = true;
return Cloud.privatePostAsync("pokerepo", data)
.then(resp => {
console.log(resp);
});
}
exports.pokeRepoAsync = pokeRepoAsync;
function apiAsync(path, postArguments) {
if (postArguments == "delete") {
return Cloud.privateDeleteAsync(path)
.then(resp => console.log(resp));
}
if (postArguments == "-") {
return nodeutil.readResAsync(process.stdin)
.then(buf => buf.toString("utf8"))
.then(str => apiAsync(path, str));
}
if (postArguments && fs.existsSync(postArguments))
postArguments = fs.readFileSync(postArguments, "utf8");
let dat = postArguments ? JSON.parse(postArguments) : null;
if (dat)
console.log("POST", "/api/" + path, JSON.stringify(dat, null, 2));
return Cloud.privateRequestAsync({
url: path,
data: dat
})
.then(resp => {
if (resp.json)
console.log(JSON.stringify(resp.json, null, 2));
else
console.log(resp.text);
});
}
exports.apiAsync = apiAsync;
function uploadFileAsync(parsed) {
const path = parsed.args[0];
let buf = fs.readFileSync(path);
let mime = U.getMime(path);
console.log("Upload", path);
return Cloud.privatePostAsync("upload/files", {
filename: path,
encoding: "base64",
content: buf.toString("base64"),
contentType: mime
})
.then(resp => {
console.log(resp);
});
}
let readlineCount = 0;
function readlineAsync() {
process.stdin.resume();
process.stdin.setEncoding('utf8');
readlineCount++;
return new Promise((resolve, reject) => {
process.stdin.once('data', (text) => {
resolve(text);
});
});
}
function queryAsync(msg, defl) {
process.stdout.write(`${msg} [${defl}]: `);
return readlineAsync()
.then(text => {
text = text.trim();
if (!text)
return defl;
else
return text;
});
}
exports.queryAsync = queryAsync;
function yesNoAsync(msg) {
process.stdout.write(msg + " (y/n): ");
return readlineAsync()
.then(text => {
if (text.trim().toLowerCase() == "y")
return Promise.resolve(true);
else if (text.trim().toLowerCase() == "n")
return Promise.resolve(false);
else
return yesNoAsync(msg);
});
}
exports.yesNoAsync = yesNoAsync;
function onlyExts(files, exts) {
return files.filter(f => exts.indexOf(path.extname(f)) >= 0);
}
function pxtFileList(pref) {
let allFiles = nodeutil.allFiles(pref + "webapp/public")
.concat(onlyExts(nodeutil.allFiles(pref + "built/web", { maxDepth: 1 }), [".js", ".css"]))
.concat(nodeutil.allFiles(pref + "built/web/fonts", { maxDepth: 1 }))
.concat(nodeutil.allFiles(pref + "built/web/vs", { maxDepth: 4 }));
for (const subapp of subwebapp_1.SUB_WEBAPPS) {
allFiles = allFiles.concat(nodeutil.allFiles(pref + `built/web/${subapp.name}`, { maxDepth: 4 }));
}
return allFiles;
}
function checkIfTaggedCommitAsync() {
return nodeutil.gitInfoAsync(["tag", "--points-at", "HEAD"])
.then(info => {
return info
.split("\n")
.some(tag => /^v\d+\.\d+\.\d+$/.test(tag.trim()));
});
}
let readJson = nodeutil.readJson;
async function ciAsync(parsed) {
const intentToPublish = parsed && parsed.flags["publish"];
const tagOverride = parsed && parsed.flags["tag"];
forceCloudBuild = true;
const buildInfo = await ciBuildInfoAsync();
pxt.log(`ci build using ${buildInfo.ci}`);
if (!buildInfo.tag)
buildInfo.tag = "";
if (!buildInfo.branch)
buildInfo.branch = "local";
let { tag, branch, pullRequest } = buildInfo;
if (tagOverride) {
tag = tagOverride;
pxt.log(`overriding tag to ${tag}`);
}
const atok = process.env.NPM_ACCESS_TOKEN;
const npmPublish = (intentToPublish || /^v\d+\.\d+\.\d+$/.exec(tag)) && atok;
if (npmPublish) {
let npmrc = path.join(process.env.HOME, ".npmrc");
pxt.log(`setting up ${npmrc} for publish`);
let cfg = "//registry.npmjs.org/:_authToken=" + atok + "\n";
fs.writeFileSync(npmrc, cfg);
}
else if (intentToPublish) {
pxt.log("not publishing, no tag or access token");
}
process.env["PXT_ENV"] = "production";
const latest = branch == "master" ? "latest" : "git-" + branch;
// upload locs on build on master
const masterOrReleaseBranchRx = /^(master|v\d+\.\d+\.\d+)$/;
const apiStringBranchRx = pxt.appTarget.uploadApiStringsBranchRx
? new RegExp(pxt.appTarget.uploadApiStringsBranchRx)
: masterOrReleaseBranchRx;
const uploadDocs = !pullRequest
&& !!pxt.appTarget.uploadDocs
&& masterOrReleaseBranchRx.test(branch);
const uploadApiStrings = !pullRequest
&& (!!pxt.appTarget.uploadDocs || pxt.appTarget.uploadApiStringsBranchRx)
&& apiStringBranchRx.test(branch);
pxt.log(`tag: ${tag}`);
pxt.log(`branch: ${branch}`);
pxt.log(`latest: ${latest}`);
pxt.log(`pull request: ${pullRequest}`);
pxt.log(`upload api strings: ${uploadApiStrings}`);
pxt.log(`upload docs: ${uploadDocs}`);
lintJSONInDirectory(path.resolve("."));
lintJSONInDirectory(path.resolve("docs"));
function npmPublishAsync() {
if (!npmPublish)
return Promise.resolve();
return nodeutil.runNpmAsync("publish");
}
let pkg = readJson("package.json");
if (pkg["name"] == "pxt-core") {
pxt.log("pxt-core build");
const isTaggedCommit = await checkIfTaggedCommitAsync();
pxt.log(`is tagged commit: ${isTaggedCommit}`);
await npmPublishAsync();
if (branch === "master" && (intentToPublish || isTaggedCommit)) {
if (uploadDocs) {
await buildWebStringsAsync();
await crowdin.uploadBuiltStringsAsync("built/webstrings.json");
for (const subapp of subwebapp_1.SUB_WEBAPPS) {
await crowdin.uploadBuiltStringsAsync(`built/${subapp.name}-strings.json`);
}
}
if (uploadApiStrings) {
await crowdin.uploadBuiltStringsAsync("built/strings.json");
}
if (uploadDocs || uploadApiStrings) {
await crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs);
pxt.log("translations uploaded");
}
else {
pxt.log("skipping translations upload");
}
}
}
else {
pxt.log("target build");
await internalBuildTargetAsync();
await internalCheckDocsAsync(true);
await blockTestsAsync();
await npmPublishAsync();
if (!process.env["PXT_ACCESS_TOKEN"]) {
// pull request, don't try to upload target
pxt.log('no token, skipping upload');
return;
}
const trg = readLocalPxTarget();
const label = `${trg.id}/${tag || latest}`;
pxt.log(`uploading target with label ${label}...`);
await uploadTargetAsync(label);
pxt.log("target uploaded");
if (uploadDocs || uploadApiStrings) {
await crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs);
pxt.log("translations uploaded");
}
else {
pxt.log("skipping translations upload");
}
}
}
function lintJSONInDirectory(dir) {
for (const file of fs.readdirSync(dir)) {
const fullPath = path.join(dir, file);
if (file.endsWith(".json")) {
const contents = fs.readFileSync(fullPath, "utf8");
try {
JSON.parse(contents);
}
catch (e) {
console.log("Could not parse " + fullPath);
process.exit(1);
}
}
}
}
function bumpPxtCoreDepAsync() {
let pkg = readJson("package.json");
if (pkg["name"] == "pxt-core")
return Promise.resolve(pkg);
let gitPull = Promise.resolve();
let commitMsg = "";
["pxt-core", "pxt-common-packages"].forEach(knownPackage => {
const modulePath = path.join("node_modules", knownPackage);
if (fs.existsSync(path.join(modulePath, ".git"))) {
gitPull = gitPull.then(() => nodeutil.spawnAsync({
cmd: "git",
args: ["pull"],
cwd: modulePath
}));
}
// not referenced
if (!fs.existsSync(path.join(modulePath, "package.json")))
return;
gitPull
.then(() => {
let kspkg = readJson(path.join(modulePath, "package.json"));
let currVer = pkg["dependencies"][knownPackage];
if (!currVer)
return; // not referenced
let newVer = kspkg["version"];
if (currVer == newVer) {
console.log(`Referenced ${knownPackage} dep up to date: ${currVer}`);
return;
}
console.log(`Bumping ${knownPackage} dep version: ${currVer} -> ${newVer}`);
if (currVer != "*" && pxt.semver.strcmp(currVer, newVer) > 0) {
U.userError(`Trying to downgrade ${knownPackage}.`);
}
if (currVer != "*" && pxt.semver.majorCmp(currVer, newVer) < 0) {
U.userError(`Trying to automatically update major version, please edit package.json manually.`);
}
pkg["dependencies"][knownPackage] = newVer;
nodeutil.writeFileSync("package.json", nodeutil.stringify(pkg) + "\n");
commitMsg += `${commitMsg ? ", " : ""}bump ${knownPackage} to ${newVer}`;
});
});
gitPull = gitPull
.then(() => commitMsg ? nodeutil.runGitAsync("commit", "-m", commitMsg, "--", "package.json") : Promise.resolve());
return gitPull;
}
function updateAsync() {
return Promise.resolve()
.then(() => nodeutil.runGitAsync("pull"))
.then(() => bumpPxtCoreDepAsync())
.then(() => nodeutil.runNpmAsync("install"));
}
async function justBumpPkgAsync(bumpType, tagCommit = true) {
ensurePkgDir();
await nodeutil.needsGitCleanAsync();
await mainPkg.loadAsync();
const curVer = pxt.semver.parse(mainPkg.config.version);
const newVer = pxt.semver.bump(curVer, bumpType);
mainPkg.config.version = await queryAsync("New version", pxt.semver.stringify(newVer));
mainPkg.saveConfig();
await nodeutil.runGitAsync("commit", "-a", "-m", mainPkg.config.version);
if (tagCommit) {
await nodeutil.runGitAsync("tag", "v" + mainPkg.config.version);
}
return mainPkg.config.version;
}
function tagReleaseAsync(parsed) {
const tag = parsed.args[0];
const version = parsed.args[1];
const npm = !!parsed.flags["npm"];
// check that ...-ref.json exists for that tag
const fn = path.join('docs', tag + "-ref.json");
pxt.log(`checking ${fn}`);
if (!fn)
U.userError(`file ${fn} does not exist`);
const v = pxt.semver.normalize(version);
const npmPkg = `pxt-${pxt.appTarget.id}`;
if (!pxt.appTarget.appTheme.githubUrl)
U.userError('pxtarget theme missing "githubUrl" entry');
// check that tag exists in github
pxt.log(`checking github ${pxt.appTarget.appTheme.githubUrl} tag v${v}`);
return U.requestAsync({
url: pxt.appTarget.appTheme.githubUrl.replace(/\/$/, '') + "/releases/tag/v" + v,
method: "GET"
})
// check that release exists in npm
.then(() => {
if (!npm)
return Promise.resolve();
pxt.log(`checking npm ${npmPkg} release`);
return nodeutil.npmRegistryAsync(npmPkg)
.then(registry => {
// verify that npm version exists
if (!registry.versions[v])
U.userError(`cannot find npm package ${npmPkg}@${v}`);
const npmTag = tag == "index" ? "latest" : tag;
return nodeutil.runNpmAsync(`dist-tag`, `add`, `${npmPkg}@v${v}`, npmTag);
});
})
// all good update ref file
.then(() => {
// update index file
nodeutil.writeFileSync(fn, JSON.stringify({
"appref": "v" + v
}, null, 4));
// TODO commit changes
console.log(`please commit ${fn} changes`);
});
}
async function bumpAsync(parsed) {
const bumpPxt = parsed && parsed.flags["update"];
const upload = parsed && parsed.flags["upload"];
let pr = parsed && parsed.flags["pr"];
let nopr = parsed && parsed.flags["nopr"];
let bumpType = parsed && parsed.flags["version"];
if (!bumpType)
bumpType = "patch";
if (bumpType.startsWith("v"))
bumpType = bumpType.slice(1);
let currBranchName = "";
let token = "";
let user = "";
let owner = "";
let repo = "";
let branchProtected = false;
try {
currBranchName = await nodeutil.getCurrentBranchNameAsync();
token = await nodeutil.getGitHubTokenAsync();
user = await nodeutil.getGitHubUserAsync(token);
({ owner, repo } = await nodeutil.getGitHubOwnerAndRepoAsync());
branchProtected = await nodeutil.isBranchProtectedAsync(token, owner, repo, currBranchName);
if (branchProtected) {
console.log(chalk.yellow(`Branch ${currBranchName} is protected.`));
pr = true;
}
}
catch (e) {
console.warn(chalk.yellow("Unable to determine branch protection status."), e.message);
}
if (nopr) {
pr = false;
}
if (fs.existsSync(pxt.CONFIG_NAME)) {
if (upload) {
U.userError("upload only supported on targets");
}
if (pr) {
console.log("Bumping via pull request.");
const newBranchName = `${user}/pxt-bump-${nodeutil.timestamp()}`;
return Promise.resolve()
.then(() => nodeutil.needsGitCleanAsync())
.then(() => nodeutil.runGitAsync("pull"))
.then(() => nodeutil.createBranchAsync(newBranchName))
.then(() => justBumpPkgAsync(bumpType, false)) // don't tag when creating a PR
.then((version) => nodeutil.gitPushAsync().then(() => version))
.then((version) => nodeutil.createPullRequestAsync({
title: `[pxt-cli] bump version to ${version}`,
body: "__Do not edit the PR title.__\n" +
"It was automatically generated by `pxt bump` and must follow a specific pattern.\n" +
"GitHub workflows rely on it to trigger version tagging and publishing to npm.",
head: newBranchName,
base: currBranchName,
token,
owner,
repo,
}))
.then((prUrl) => nodeutil.switchBranchAsync(currBranchName).then(() => prUrl))
.then((prUrl) => console.log(`${chalk.green('PR created:')} ${chalk.green.underline(prUrl)}`));
}
else {
console.log("Bumping via direct push.");
return Promise.resolve()
.then(() => nodeutil.runGitAsync("pull"))
.then(() => justBumpPkgAsync(bumpType))
.then(() => nodeutil.runGitAsync("push", "--follow-tags"))
.then(() => nodeutil.runGitAsync("push"));
}
}
else if (fs.existsSync("pxtarget.json"))
if (pr) {
console.log("Bumping via pull request.");
const newBranchName = `${user}/pxt-bump-${nodeutil.timestamp()}`;
return Promise.resolve()
.then(() => nodeutil.needsGitCleanAsync())
.then(() => nodeutil.runGitAsync("pull"))
.then(() => nodeutil.createBranchAsync(newBranchName))
.then(() => bumpPxt ? bumpPxtCoreDepAsync() : Promise.resolve())
.then(() => nodeutil.npmVersionBumpAsync(bumpType, false)) // don't tag when creating a PR
.then((version) => nodeutil.gitPushAsync().then(() => version))
.then((version) => nodeutil.createPullRequestAsync({
title: `[pxt-cli] bump version to ${version}`,
body: "__Do not edit the PR title.__\n" +
"It was automatically generated by `pxt bump` and must follow a specific pattern.\n" +
"GitHub workflows rely on it to trigger version tagging and publishing to npm.",
head: newBranchName,
base: currBranchName,
token,
owner,
repo,
}))
.then((prUrl) => nodeutil.switchBranchAsync(currBranchName).then(() => prUrl))
.then((prUrl) => console.log(`${chalk.green('PR created:')} ${chalk.green.underline(prUrl)}`));
}
else {
console.log("Bumping via direct push.");
return Promise.resolve()
.then(() => nodeutil.runGitAsync("pull"))
.then(() => bumpPxt ? bumpPxtCoreDepAsync().then(() => nodeutil.runGitAsync("push")) : Promise.resolve())
.then(() => nodeutil.runNpmAsync("version", bumpType))
.then(() => nodeutil.runGitAsync("push", "--follow-tags"))
.then(() => nodeutil.runGitAsync("push"))
.then(() => upload ? uploadTaggedTargetAsync() : Promise.resolve());
}
else {
U.userError("Couldn't find package or target JSON file; nothing to bump");
}
}
function uploadTaggedTargetAsync() {
forceCloudBuild = true;
if (!pxt.github.token) {
fatal("GitHub token not found, please use 'pxt login' to login with your GitHub account to push releases.");
return Promise.resolve();
}
return nodeutil.needsGitCleanAsync()
.then(() => Promise.all([
nodeutil.currGitTagAsync(),
nodeutil.gitInfoAsync(["rev-parse", "--abbrev-ref", "HEAD"]),
nodeutil.gitInfoAsync(["rev-parse", "HEAD"])
]))
// only build target after getting all the info
.then(info => internalBuildTargetAsync()
.then(() => internalCheckDocsAsync(true))
.then(() => info))
.then(async (info) => {
const repoSlug = "microsoft/pxt-" + pxt.appTarget.id;
setCiBuildInfo(info[0], info[1], info[2], repoSlug);
process.env['PXT_RELEASE_REPO'] = "https://git:" + pxt.github.token + "@github.com/" + repoSlug + "-built";
let v = await pkgVersionAsync();
pxt.log("uploading " + v);
return uploadCoreAsync({
label: "v" + v,
fileList: pxtFileList("node_modules/pxt-core/").concat(targetFileList()),
pkgversion: v,
githubOnly: true,
fileContent: {}
});
});
}
async function pkgVersionAsync() {
let ver = readJson("package.json")["version"];
const info = await ciBuildInfoAsync();
if (!info.tag)
ver += "-" + (info.commit ? info.commit.slice(0, 6) : "local");
return ver;
}
function targetFileList() {
let lst = onlyExts(nodeutil.allFiles("built"), [".js", ".css", ".json", ".webmanifest"])
.concat(nodeutil.allFiles(path.join(simDir(), "public")));
if (simDir() != "sim")
lst = lst.concat(nodeutil.allFiles(path.join("sim", "public"), { maxDepth: 5, allowMissing: true }));
pxt.debug(`target files (on disk): ${lst.join('\r\n ')}`);
return lst;
}
async function uploadTargetAsync(label) {
return uploadCoreAsync({
label,
fileList: pxtFileList("node_modules/pxt-core/").concat(targetFileList()),
pkgversion: await pkgVersionAsync(),
fileContent: {}
});
}
function uploadTargetReleaseAsync(parsed) {
parseBuildInfo(parsed);
const label = parsed.args[0];
const rebundle = !!parsed.flags["rebundle"];
return (rebundle ? rebundleAsync() : internalBuildTargetAsync())
.then(() => {
return uploadTargetAsync(label);
});
}
exports.uploadTargetReleaseAsync = uploadTargetReleaseAsync;
function uploadTargetRefsAsync(repoPath) {
if (repoPath)
process.chdir(repoPath);
return nodeutil.needsGitCleanAsync()
.then(() => Promise.all([
nodeutil.gitInfoAsync(["rev-parse", "HEAD"]),
nodeutil.gitInfoAsync(["config", "--get", "remote.origin.url"])
]))
.then(info => {
return gitfs.uploadRefs(info[0], info[1])
.then(() => {
return Promise.resolve();
});
});
}
exports.uploadTargetRefsAsync = uploadTargetRefsAsync;
function uploadFileName(p) {
// normalize /, \ before filtering
return p.replace(/\\/g, '\/')
.replace(/^.*(built\/web\/|\w+\/public\/|built\/)/, "");
}
function gitUploadAsync(opts, uplReqs) {
let reqs = U.unique(U.values(uplReqs), r => r.hash);
console.log("Asking for", reqs.length, "hashes");
return Promise.resolve()
.then(() => Cloud.privatePostAsync("upload/status", {
hashes: reqs.map(r => r.hash)
}))
.then(resp => {
let missing = U.toDictionary(resp.missing, s => s);
let missingReqs = reqs.filter(r => !!U.lookup(missing, r.hash));
let size = 0;
for (let r of missingReqs)
size += r.size;
console.log("files missing: ", missingReqs.length, size, "bytes");
return U.promiseMapAll(missingReqs, r => Cloud.privatePostAsync("upload/blob", r)
.then(() => {
console.log(r.filename + ": OK," + r.size + " " + r.hash);
}));
})
.then(async () => {
let roottree = {};
let get = (tree, path) => {
let subt = U.lookup(tree, path);
if (!subt)
subt = tree[path] = {};
return subt;
};
let lookup = (tree, path) => {
let m = /^([^\/]+)\/(.*)/.exec(path);
if (m) {
let subt = get(tree, m[1]);
U.assert(!subt.hash);
if (!subt.subtree)
subt.subtree = {};
return lookup(subt.subtree, m[2]);
}
else {
return get(tree, path);
}
};
for (let fn of Object.keys(uplReqs)) {
let e = lookup(roottree, fn);
e.hash = uplReqs[fn].hash;
}
const info = await ciBuildInfoAsync();
let data = {
message: "Upload from " + info.commitUrl,
parents: [],
target: pxt.appTarget.id,
tree: roottree,
};
console.log("Creating commit...");
return Cloud.privatePostAsync("upload/commit", data);
})
.then(res => {
console.log("Commit:", res);
return uploadToGitRepoAsync(opts, uplReqs);
});
}
async function uploadToGitRepoAsync(opts, uplReqs) {
let label = opts.label;
if (!label) {
console.log('no label; skip release upload');
return Promise.resolve();
}
let tid = pxt.appTarget.id;
if (U.startsWith(label, tid + "/"))
label = label.slice(tid.length + 1);
if (!/^v\d/.test(label)) {
console.log(`label "${label}" is not a version; skipping release upload`);
return Promise.resolve();
}
let repoUrl = process.env["PXT_RELEASE_REPO"];
if (!repoUrl) {
console.log("no $PXT_RELEASE_REPO variable; not uploading label " + label);
return Promise.resolve();
}
nodeutil.mkdirP("tmp");
let trgPath = "tmp/releases";
let mm = /^https:\/\/([^:]+):([^@]+)@([^\/]+)(.*)/.exec(repoUrl);
if (!mm) {
U.userError("wrong format for $PXT_RELEASE_REPO");
}
console.log(`create release ${label} in ${repoUrl}`);
let user = mm[1];
let pass = mm[2];
let host = mm[3];
let netRcLine = `machine ${host} login ${user} password ${pass}\n`;
repoUrl = `https://${user}@${host}${mm[4]}`;
let homePath = process.env["HOME"] || process.env["UserProfile"];
let netRcPath = path.join(homePath, /^win/.test(process.platform) ? "_netrc" : ".netrc");
let prevNetRc = fs.existsSync(netRcPath) ? fs.readFileSync(netRcPath, "utf8") : null;
let newNetRc = prevNetRc ? prevNetRc + "\n" + netRcLine : netRcLine;
console.log("Adding credentials to " + netRcPath);
fs.writeFileSync(netRcPath, newNetRc, {
encoding: "utf8",
mode: '600'
});
let cuser = process.env["USER"] || "";
if (cuser && !/travis/.test(cuser))
user += "-" + cuser;
const cred = [
"-c", "credential.helper=",
"-c", "user.name=" + user,
"-c", "user.email=" + user + "@build.pxt.io",
];
const gitAsync = (args) => nodeutil.spawnAsync({
cmd: "git",
cwd: trgPath,
args: cred.concat(args)
});
const info = await ciBuildInfoAsync();
return Promise.resolve()
.then(() => {
if (fs.existsSync(trgPath)) {
let cfg = fs.readFileSync(trgPath + "/.git/config", "utf8");
if (cfg.indexOf("url = " + repoUrl) > 0) {
return gitAsync(["pull", "--depth=3"]);
}
else {
U.userError(trgPath + " already exists; please remove it");
}
}
else {
return nodeutil.spawnAsync({
cmd: "git",
args: cred.concat(["clone", "--depth", "3", repoUrl, trgPath]),
cwd: "."
});
}
})
.then(() => {
for (let u of U.values(uplReqs)) {
let fpath = path.join(trgPath, u.filename);
nodeutil.mkdirP(path.dirname(fpath));
fs.writeFileSync(fpath, u.content, { encoding: u.encoding });
}
// make sure there's always something to commit
fs.writeFileSync(trgPath + "/stamp.txt", new Date().toString());
})
.then(() => gitAsync(["add", "."]))
.then(() => gitAsync(["commit", "-m", "Release " + label + " from " + info.commitUrl]))
.then(() => gitAsync(["tag", label]))
.then(() => gitAsync(["push"]))
.then(() => gitAsync(["push", "--tags"]))
.then(() => {
})
.finally(() => {
if (prevNetRc == null) {
console.log("Removing " + netRcPath);
fs.unlinkSync(netRcPath);
}
else {
console.log("Restoring " + netRcPath);
fs.writeFileSync(netRcPath, prevNetRc, {
mode: '600'
});
}
});
}
function uploadedArtFileCdnUrl(fn) {
if (!fn || /^(https?|data):/.test(fn))
return fn; // nothing to do
fn = fn.replace(/^\.?\/*/, "/");
const cdnBlobUrl = "@cdnUrl@/blob/" + gitHash(fs.readFileSync("docs" + fn)) + "" + fn;
return cdnBlobUrl;
}
function gitHash(buf) {
let hash = crypto.createHash("sha1");
hash.update(Buffer.from("blob " + buf.length + "\u0000", "utf8"));
hash.update(buf);
return hash.digest("hex");
}
function uploadCoreAsync(opts) {
var _a, _b;
const targetConfig = readLocalPxTarget();
const defaultLocale = targetConfig.appTheme.defaultLocale;
const hexCache = path.join("built", "hexcache");
let hexFiles = [];
if (fs.existsSync(hexCache)) {
hexFiles = fs.readdirSync(hexCache)
.filter(f => /\.hex$/.test(f))
.filter(f => fs.readFileSync(path.join(hexCache, f), { encoding: "utf8" }) != "SKIP")
.map((f) => `@cdnUrl@/compile/${f}`);
pxt.log(`hex cache:\n\t${hexFiles.join('\n\t')}`);
}
const targetUsedImages = {};
const cdnCachedAppTheme = replaceStaticImagesInJsonBlob(readLocalPxTarget(), fn => {
const fp = path.join("docs", fn);
if (!targetUsedImages[fn] && !opts.fileList.includes(fp)) {
opts.fileList.push(fp);
}
targetUsedImages[fn] = uploadedArtFileCdnUrl(fn);
return targetUsedImages[fn];
}).appTheme;
const targetImagePaths = Object.keys(targetUsedImages);
const targetImagesHashed = Object.values(targetUsedImages);
let targetEditorJs = "";
if ((_a = pxt.appTarget.appTheme) === null || _a === void 0 ? void 0 : _a.extendEditor)
targetEditorJs = "@commitCdnUrl@editor.js";
let targetFieldEditorsJs = "";
if ((_b = pxt.appTarget.appTheme) === null || _b === void 0 ? void 0 : _b.extendFieldEditors)
targetFieldEditorsJs = "@commitCdnUrl@fieldeditors.js";
let replacements = {
"/sim/simulator.html": "@simUrl@",
"/sim/siminstructions.html": "@partsUrl@",
"/sim/sim.webmanifest": "@relprefix@webmanifest",
"/embed.js": "@targetUrl@@relprefix@embed",
"/cdn/": "@commitCdnUrl@",
"/doccdn/": "@commitCdnUrl@",
"/sim/": "@commitCdnUrl@",
"/blb/": "@blobCdnUrl@",
"/trgblb/": "@targetBlobUrl@",
"@timestamp@": "",
"data-manifest=\"\"": "@manifest@",
"var pxtConfig = null": "var pxtConfig = @cfg@",
"var pxtConfig=null": "var pxtConfig = @cfg@",
"@defaultLocaleStrings@": defaultLocale ? "@commitCdnUrl@" + "locales/" + defaultLocale + "/strings.json" : "",
"@cachedHexFiles@": hexFiles.length ? hexFiles.join("\n") : "",
"@cachedHexFilesEncoded@": encodeURLs(hexFiles),
"@targetEditorJs@": targetEditorJs,
"@targetFieldEditorsJs@": targetFieldEditorsJs,
"@targetImages@": targetImagesHashed.length ? targetImagesHashed.join('\n') : '',
"@targetImagesEncoded@": targetImagesHashed.length ? encodeURLs(targetImagesHashed) : ""
};
if (opts.localDir) {
let cfg = {
"relprefix": opts.localDir,
"verprefix": "",
"workerjs": opts.localDir + "worker.js",
"monacoworkerjs": opts.localDir + "monacoworker.js",
"gifworkerjs": opts.localDir + "gifjs/gif.worker.js",
"serviceworkerjs": opts.localDir + "serviceworker.js",
"typeScriptWorkerJs": opts.localDir + "tsworker.js",
"pxtVersion": pxtVersion(),
"pxtRelId": "localDirRelId",
"pxtCdnUrl": opts.localDir,
"commitCdnUrl": opts.localDir,
"blobCdnUrl": opts.localDir,
"cdnUrl": opts.localDir,
"targetVersion": opts.pkgversion,
"targetRelId": "",
"targetUrl": "",
"targetId": opts.target,
"simUrl": opts.localDir + "simulator.html",
"simserviceworkerUrl": opts.localDir + "simulatorserviceworker.js",
"simworkerconfigUrl": opts.localDir + "workerConfig.js",
"partsUrl": opts.localDir + "siminstructions.html",
"runUrl": opts.localDir + "run.html",
"docsUrl": opts.localDir + "docs.html",
"multiUrl": opts.localDir + "multi.html",
"asseteditorUrl": opts.localDir + "asseteditor.html",
"isStatic": true,
};
for (const subapp of subwebapp_1.SUB_WEBAPPS) {
cfg[subapp.name + "Url"] = opts.localDir + subapp.name + ".html";
}
const targetImageLocalPaths = targetImagePaths.map(k => `${opts.localDir}${path.join('./docs', k)}`);
replacements = {
"/embed.js": opts.localDir + "embed.js",
"/cdn/": opts.localDir,
"/doccdn/": opts.localDir,
"/sim/": opts.localDir,
"/blb/": opts.localDir,
"/trgblb/": opts.localDir,
"@monacoworkerjs@": `${opts.localDir}monacoworker.js`,
"@gifworkerjs@": `${opts.localDir}gifjs/gif.worker.js`,
"@workerjs@": `${opts.localDir}worker.js`,
"@serviceworkerjs@": `${opts.localDir}serviceworker.js`,
"@timestamp@": `# ver ${new Date().toString()}`,
"var pxtConfig = null": "var pxtConfig = " + JSON.stringify(cfg, null, 4),
"@defaultLocaleStrings@": "",
"@cachedHexFiles@": "",
"@cachedHexFilesEncoded@": "",
"@targetEditorJs@": targetEditorJs ? `${opts.localDir}editor.js` : "",
"@targetFieldEditorsJs@": targetFieldEditorsJs ? `${opts.localDir}fieldeditors.js` : "",
"@targetImages@": targetImagePaths.length ? targetImageLocalPaths.join('\n') : '',
"@targetImagesEncoded@": targetImagePaths.length ? encodeURLs(targetImageLocalPaths) : ''
};
if (!opts.noAppCache) {
replacements["data-manifest=\"\""] = `manifest="${opts.localDir}release.manifest"`;
}
}
const replFiles = [
"index.html",
"embed.js",
"run.html",
"docs.html",
"siminstructions.html",
"codeembed.html",
"release.manifest",
"worker.js",
"serviceworker.js",
"simulatorserviceworker.js",
"monacoworker.js",
"simulator.html",
"sim.manifest",
"sim.webmanifest",
"workerConfig.js",
"multi.html",
"asseteditor.html"
];
// expandHtml is manually called on these files before upload
// runs <!-- @include --> substitutions, fills in locale, etc
const expandFiles = [
"index.html"
];
for (const subapp of subwebapp_1.SUB_WEBAPPS) {
replFiles.push(`${subapp.name}.html`);
expandFiles.push(`${subapp.name}.html`);
}
nodeutil.mkdirP("built/uploadrepl");
function encodeURLs(urls) {
return urls.map(url => encodeURIComponent(url)).join(";");
}
const uplReqs = {};
const uglify = opts.minify ? require("uglify-js") : undefined;
const uploadFileAsync = async (p) => {
var _a, _b;
let rdata = null;
if (opts.fileContent) {
let s = U.lookup(opts.fileContent, p);
if (s != null)
rdata = Buffer.from(s, "utf8");
}
if (!rdata) {
if (!fs.existsSync(p))
return undefined;
rdata = await readFileAsync(p);
}
let fileName = uploadFileName(p);
let mime = U.getMime(p);
const minified = opts.minify && mime == "application/javascript" && fileName !== "target.js";
pxt.log(` ${p} -> ${fileName} (${mime})` + (minified ? ' minified' : ""));
let isText = /^(text\/.*|application\/.*(javascript|json))$/.test(mime);
let content = "";
let data = rdata;
if (isText) {
content = data.toString("utf8");
if (expandFiles.indexOf(fileName) >= 0) {
if (!opts.localDir) {
let m = pxt.appTarget.appTheme;
for (let k of Object.keys(m)) {
if (/CDN$/.test(k))
m[k.slice(0, k.length - 3)] = m[k];
}
}
content = server.expandHtml(content, undefined, cdnCachedAppTheme);
}
if (/^sim/.test(fileName) || /^workerConfig/.test(fileName)) {
// just force blobs for everything in simulator manifest
content = content.replace(/\/(cdn|sim)\//g, "/blb/");
}
if (minified) {
const res = uglify.minify(content);
if (!res.error) {
content = res.code;
}
else {
pxt.log(` Could not minify ${fileName} ${res.error}`);
}
}
if (replFiles.indexOf(fileName) >= 0) {
for (let from of Object.keys(replacements)) {
content = U.replaceAll(content, from, replacements[from]);
}
if (opts.localDir) {
data = Buffer.from(content, "utf8");
}
else {
// save it for developer inspection
fs.writeFileSync("built/uploadrepl/" + fileName, content);
}
}
else if (fileName == "target.json" || fileName == "target.js") {
let isJs = fileName == "target.js";
if (isJs)
content = content.slice(targetJsPrefix.length);
let trg = JSON.parse(content);
if (opts.localDir) {
for (let e of trg.appTheme.docMenu)
if (e.path[0] == "/") {
e.path = opts.localDir + "docs" + e.path;
}
trg.appTheme.homeUrl = opts.localDir;
// patch icons in bundled packages
Object.keys(trg.bundledpkgs).forEach(pkgid => {
const res = trg.bundledpkgs[pkgid];
// path config before storing
const config = JSON.parse(res[pxt.CONFIG_NAME]);
if (/^\//.test(config.icon))
config.icon = opts.localDir + "docs" + config.icon;
res[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(config);
});
data = Buffer.from((isJs ? targetJsPrefix : '') + nodeutil.stringify(trg), "utf8");
}
else {
if ((_b = (_a = trg.simulator) === null || _a === void 0 ? void 0 : _a.boardDefinition) === null || _b === void 0 ? void 0 : _b.visual) {
let boardDef = trg.simulator.boardDefinition.visual;
if (boardDef.image) {
boardDef.image = uploadedArtFileCdnUrl(boardDef.image);
if (boardDef.outlineImage)
boardDef.outlineImage = uploadedArtFileCdnUrl(boardDef.outlineImage);
}
}
// patch icons in bundled packages
Object.keys(trg.bundledpkgs).forEach(pkgid => {
const res = trg.bundledpkgs[pkgid];
// patch config before storing
const config = JSON.parse(res[pxt.CONFIG_NAME]);
if (config.icon)
config.icon = uploadedArtFileCdnUrl(config.icon);
res[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(config);
});
content = nodeutil.stringify(trg);
if (isJs)
content = targetJsPrefix + content;
// save it for developer inspection
fs.writeFileSync("built/uploadrepl/" + fileName, content);
}
}
}
else {
content = data.toString("base64");
}
if (opts.localDir) {
U.assert(!!opts.builtPackaged);
let fn = path.join(opts.builtPackaged, opts.localDir, fileName);
nodeutil.mkdirP(path.dirname(fn));
return minified ? writeFileAsync(fn, content) : writeFileAsync(fn, data);
}
let req = {
encoding: isText ? "utf8" : "base64",
content,
hash: "",
filename: fileName,
size: 0
};
let buf = Buffer.from(req.content, req.encoding);
req.size = buf.length;
req.hash = gitHash(buf);
uplReqs[fileName] = req;
};
// only keep the last version of each uploadFileName()
opts.fileList = U.values(U.toDictionary(opts.fileList, uploadFileName));
// check size
const maxSize = checkFileSize(opts.fileList);
const maxAllowedFileSize = (pxt.appTarget.cloud.maxFileSize || (30000000)); // default to 30Mb
if (maxSize > maxAllowedFileSize)
U.userError(`file too big for upload: ${maxSize} bytes, max is ${maxAllowedFileSize} bytes`);
pxt.log('');
if (opts.localDir)
return U.promisePoolAsync(15, opts.fileList, uploadFileAsync)
.then(() => {
pxt.log("Release files written to " + path.join(opts.builtPackaged, opts.localDir));
});
return U.promisePoolAsync(15, opts.fileList, uploadFileAsync)
.then(() => opts.githubOnly
? uploadToGitRepoAsync(opts, uplReqs)
: gitUploadAsync(opts, uplReqs));
}
function readLocalPxTarget() {
if (!fs.existsSync("pxtarget.json")) {
console.error("This command requires pxtarget.json in current directory.");
process.exit(1);
}
nodeutil.setTargetDir(process.cwd());
const pkg = readJson("package.json");
const cfg = readJson("pxtarget.json");
cfg.versions = {
target: pkg["version"],
pxt: (pxt.appTarget.versions && pxt.appTarget.versions.pxt) || pkg["dependencies"]["pxt-core"]
};
return cfg;
}
function forEachBundledPkgAsync(f, includeProjects = false) {
let prev = process.cwd();
let folders = pxt.appTarget.bundleddirs;
if (includeProjects) {
let projects = nodeutil.allFiles("libs", { maxDepth: 1, includeDirs: true }).filter(f => /prj$/.test(f));
folders = folders.concat(projects);
}
return U.promiseMapAllSeries(folders, (dirname) => {
const host = new Host();
const pkgPath = path.join(nodeutil.targetDir, dirname);
pxt.debug(`building bundled package at ${pkgPath}`);
// if the