ayakashi
Version:
The next generation web scraping framework
151 lines (150 loc) • 6.7 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getRecommendedChromiumRevision = exports.downloadChromium = void 0;
const core_1 = __importDefault(require("@ayakashi/request/core"));
const request_1 = __importDefault(require("@ayakashi/request"));
const opLog_1 = require("../opLog/opLog");
const fs_1 = require("fs");
const extract_zip_1 = __importDefault(require("extract-zip"));
const store_1 = require("../store/store");
const chromium_1 = require("../store/chromium");
function downloadChromium(options, storedRevision) {
return __awaiter(this, void 0, void 0, function* () {
const opLog = opLog_1.getOpLog();
let chromiumArch = "";
let filename = "";
if (process.platform === "win32") {
if (process.arch === "x64") {
chromiumArch = "win64";
filename = "chrome-win64";
}
else if (process.arch === "x32") {
chromiumArch = "win32";
filename = "chrome-win32";
}
}
if (process.platform === "darwin") {
if (process.arch === "x64") {
chromiumArch = "mac-x64";
filename = "chrome-mac-x64";
}
}
if (process.platform === "linux") {
if (process.arch === "x64") {
chromiumArch = "linux64";
filename = "chrome-linux64";
}
}
if (!chromiumArch) {
opLog.error("unsupported architecture:", process.platform, "-", process.arch);
throw new Error("unsupported_architecture");
}
let revision = options.revision;
if (options.useChannel) {
try {
let versions = yield request_1.default
.get(`https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json`);
versions = JSON.parse(versions);
revision = versions.channels[options.channel].version;
}
catch (_e) {
throw new Error("invalid_chromium_revision");
}
}
opLog.info("downloading chrome", `${revision}`, "for", chromiumArch);
if (storedRevision === revision) {
opLog.info(`downloaded chrome is already at revision ${revision}`);
return;
}
yield chromium_1.cleanChromiumDirectory();
const storeDir = yield store_1.getStoreDir();
return new Promise(function (resolve, reject) {
const downloadStream = core_1.default
.get(`https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${revision}/${chromiumArch}/${filename}.zip`);
let total = "0mb";
let downloaded = 0;
const waiter = opLog.waiter("0.0MB/0.0MB");
downloadStream.on("response", function (resp) {
total = toMb(parseInt(resp.headers["content-length"] || "0"));
});
const fileStream = fs_1.createWriteStream(`${storeDir}/chromium.zip`);
downloadStream.pipe(fileStream);
downloadStream.on("data", function (chunk) {
downloaded += chunk.length;
waiter.text = `${toMb(downloaded)}/${total}`;
});
downloadStream.on("end", function () {
waiter.text = "extracting...";
extract_zip_1.default(`${storeDir}/chromium.zip`, { dir: `${storeDir}/chromium` }, function (err) {
return __awaiter(this, void 0, void 0, function* () {
if (err) {
waiter.fail(err.message);
yield chromium_1.cleanChromiumDirectory();
yield cleanZipFile(`${storeDir}/chromium.zip`);
reject(err);
}
else {
yield cleanZipFile(`${storeDir}/chromium.zip`);
const chromePath = yield chromium_1.getChromePath();
fs_1.chmodSync(chromePath, 0o755);
yield chromium_1.updateStoredRevision(revision);
waiter.succeed("done!");
resolve();
}
});
});
});
downloadStream.on("error", function (err) {
return __awaiter(this, void 0, void 0, function* () {
waiter.fail(err.message);
yield cleanZipFile(`${storeDir}/chromium.zip`);
reject(err);
});
});
});
});
}
exports.downloadChromium = downloadChromium;
const RECOMMENDED_CHROMIUM_REVISION_FALLBACK = "114.0.5735.133";
function getRecommendedChromiumRevision() {
return __awaiter(this, void 0, void 0, function* () {
try {
//tslint:disable
let text = yield request_1.default
.get("https://raw.githubusercontent.com/puppeteer/puppeteer/main/packages/puppeteer-core/src/revisions.ts");
//tslint:enable
text = text.replace(/\s+/g, "");
const match = text.match(/{.*}/);
if (!match)
return RECOMMENDED_CHROMIUM_REVISION_FALLBACK;
return match[0].split(":")[1].split(",")[0].replace(/'/g, "");
}
catch (_e) {
return RECOMMENDED_CHROMIUM_REVISION_FALLBACK;
}
});
}
exports.getRecommendedChromiumRevision = getRecommendedChromiumRevision;
function toMb(bytes) {
const mb = (bytes / 1000 / 1000).toFixed(3);
return `${mb}MB`;
}
function cleanZipFile(zipPath) {
return new Promise(function (resolve) {
fs_1.unlink(zipPath, function (_err) {
resolve();
});
});
}