@wapython/wasi
Version:
Javascript library for interacting with WASI Modules in Node.js and the Browser.
293 lines (288 loc) • 13.1 kB
JavaScript
"use strict";
/*
Create a union filesystem as described by a FileSystemSpec[].
This code should not depend on anything that must run in node.js.
Note that this is entirely synchronous code, e.g., the unzip code,
and that's justified because our WASM interpreter will likely get
run in a different thread (a webworker) than the main thread, and
this code is needed to initialize it before anything else can happen.
*/
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __values = (this && this.__values) || function(o) {
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
if (m) return m.call(o);
if (o && typeof o.length === "number") return {
next: function () {
if (o && i >= o.length) o = void 0;
return { value: o && o[i++], done: !o };
}
};
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createFileSystem = void 0;
var unzip_1 = __importDefault(require("./unzip"));
var memfs_1 = require("memfs");
var unionfs_1 = require("@wapython/unionfs");
function createFileSystem(specs, nativeFs) {
var e_1, _a;
var _this = this;
var _b;
if (specs.length == 0) {
return memFs(); // empty memfs
}
if (specs.length == 1) {
// don't use unionfs:
return (_b = specToFs(specs[0], nativeFs)) !== null && _b !== void 0 ? _b : memFs();
}
var ufs = new unionfs_1.Union();
var v = [];
try {
for (var specs_1 = __values(specs), specs_1_1 = specs_1.next(); !specs_1_1.done; specs_1_1 = specs_1.next()) {
var spec = specs_1_1.value;
var fs = specToFs(spec, nativeFs);
if (fs != null) {
// e.g., native bindings may be null.
ufs.use(fs);
if (fs.waitUntilLoaded != null) {
v.push(fs.waitUntilLoaded.bind(fs));
}
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (specs_1_1 && !specs_1_1.done && (_a = specs_1.return)) _a.call(specs_1);
}
finally { if (e_1) throw e_1.error; }
}
var waitUntilLoaded = function () { return __awaiter(_this, void 0, void 0, function () {
var v_1, v_1_1, wait, e_2_1;
var e_2, _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
_b.trys.push([0, 5, 6, 7]);
v_1 = __values(v), v_1_1 = v_1.next();
_b.label = 1;
case 1:
if (!!v_1_1.done) return [3 /*break*/, 4];
wait = v_1_1.value;
return [4 /*yield*/, wait()];
case 2:
_b.sent();
_b.label = 3;
case 3:
v_1_1 = v_1.next();
return [3 /*break*/, 1];
case 4: return [3 /*break*/, 7];
case 5:
e_2_1 = _b.sent();
e_2 = { error: e_2_1 };
return [3 /*break*/, 7];
case 6:
try {
if (v_1_1 && !v_1_1.done && (_a = v_1.return)) _a.call(v_1);
}
finally { if (e_2) throw e_2.error; }
return [7 /*endfinally*/];
case 7: return [2 /*return*/];
}
});
}); };
return __assign(__assign({}, ufs), { constants: memfs_1.fs.constants, waitUntilLoaded: waitUntilLoaded });
}
exports.createFileSystem = createFileSystem;
function specToFs(spec, nativeFs) {
// All these "as any" are because really nothing quite implements FileSystem yet!
// See https://github.com/streamich/memfs/issues/735
if (spec.type == "zip") {
return zipFs(spec.data, spec.mountpoint);
}
else if (spec.type == "zip-async") {
return zipFsAsync(spec.getData, spec.mountpoint);
}
else if (spec.type == "zipfile") {
throw Error("you must convert zipfile -- read ".concat(spec.zipfile, " into memory"));
}
else if (spec.type == "zipurl") {
throw Error("you must convert zipurl -- read ".concat(spec.zipurl, " into memory"));
}
else if (spec.type == "native") {
// native = whatever is in bindings.
return nativeFs == null ? nativeFs : mapFlags(nativeFs);
}
else if (spec.type == "mem") {
return memFs(spec.contents);
}
else if (spec.type == "dev") {
return devFs();
}
throw Error("unknown spec type - ".concat(JSON.stringify(spec)));
}
// this is generic and would work in a browser:
function devFs() {
var vol = memfs_1.Volume.fromJSON({
"/dev/stdin": "",
"/dev/stdout": "",
"/dev/stderr": "",
});
vol.releasedFds = [0, 1, 2];
var fdErr = vol.openSync("/dev/stderr", "w");
var fdOut = vol.openSync("/dev/stdout", "w");
var fdIn = vol.openSync("/dev/stdin", "r");
if (fdErr != 2)
throw Error("invalid handle for stderr: ".concat(fdErr));
if (fdOut != 1)
throw Error("invalid handle for stdout: ".concat(fdOut));
if (fdIn != 0)
throw Error("invalid handle for stdin: ".concat(fdIn));
return (0, memfs_1.createFsFromVolume)(vol);
}
function zipFs(data, directory) {
if (directory === void 0) { directory = "/"; }
var fs = (0, memfs_1.createFsFromVolume)(new memfs_1.Volume());
(0, unzip_1.default)({ data: data, fs: fs, directory: directory });
return fs;
}
function zipFsAsync(getData, directory) {
var _this = this;
if (directory === void 0) { directory = "/"; }
var fs = (0, memfs_1.createFsFromVolume)(new memfs_1.Volume());
var load = function () { return __awaiter(_this, void 0, void 0, function () {
var data, err_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, getData()];
case 1:
data = _a.sent();
return [3 /*break*/, 3];
case 2:
err_1 = _a.sent();
console.warn("FAILED to load async filesystem for '".concat(directory, "' - ").concat(err_1));
throw err_1;
case 3:
// NOTE: there is an async version of this, but it runs in another
// webworker and costs significant overhead, so not worth it.
(0, unzip_1.default)({ data: data, fs: fs, directory: directory });
return [2 /*return*/];
}
});
}); };
var loadingPromise = load();
fs.waitUntilLoaded = function () { return loadingPromise; };
return fs;
}
function memFs(contents) {
var vol = contents != null ? memfs_1.Volume.fromJSON(contents) : new memfs_1.Volume();
return (0, memfs_1.createFsFromVolume)(vol);
}
function mapFlags(nativeFs) {
var _this = this;
function translate(flags) {
// We have to translate the flags from WASM/memfs/musl to native for this operating system.
// E.g., on MacOS many flags are completely different. See big comment below.
var nativeFlags = 0;
for (var flag in memfs_1.fs.constants) {
// only flags starting with O_ are relevant for the open syscall.
if (flag.startsWith("O_") && flags & memfs_1.fs.constants[flag]) {
nativeFlags |= nativeFs.constants[flag];
}
}
return nativeFlags;
}
// "any" because there's something weird involving a __promises__ namespace that I don't understand.
var open = function (path, flags, mode) { return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, nativeFs.open(path, translate(flags), mode)];
case 1: return [2 /*return*/, _a.sent()];
}
});
}); };
var openSync = function (path, flags, mode) {
return nativeFs.openSync(path, translate(flags), mode);
};
var promises = __assign(__assign({}, nativeFs.promises), { open: function (path, flags, mode) { return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, nativeFs.promises.open(path, flags, mode)];
case 1: return [2 /*return*/, _a.sent()];
}
});
}); } });
return __assign(__assign({}, __assign(__assign({}, nativeFs), { promises: promises })), { open: open, openSync: openSync });
}
/*
Comment about flags:
A major subtle issue I hit is that unionfs combines filesystems, and
each filesystem can define fs.constants differently! In particular,
memfs always hardcodes constants.O_EXCL to be 128. However, on
macos native filesystem it is 2048, whereas on Linux native filesystem
it is also 128. We combine memfs and native for running python-wasm
under nodejs, since we want to use our Python install (that is in
dist/python/python.zip and mounted using memfs) along with full access
to the native filesystem.
I think the only good solution to this is the following:
- if native isn't part of the unionfs, nothing to do (since we only currently use native and memfs).
- fs.constants should be memfs's constants since I think they match with what WebAssembly libc (via musl)
provides.
- in the node api, the ONLY functions that take numeric flags are open and openSync. That's convenient!
- somehow figure out which filesystem (native or memfs for now) that a given open will go to, and
convert the flags if going to memfs.
Probably the easiest way to accomplish all of the above is just use a proxy around native fs's
open* function.
*/
//# sourceMappingURL=fs.js.map