@velcro/strategy-cdn
Version:
Velcro resolver strategy for resolving modules from a cdn such as Unpkg or JsDelivr
441 lines (437 loc) • 17.6 kB
JavaScript
import { Uri, isThenable, all, EntryNotFoundError, basename, checkCancellation, parseBufferAsRootPackageJson } from '@velcro/common';
import { AbstractResolverStrategyWithRoot, ResolverStrategy, ResolverContext } from '@velcro/resolver';
import satisfies from 'semver/functions/satisfies';
import validRange from 'semver/ranges/valid';
function isValidEntry(entry) {
if (!entry || typeof entry !== 'object')
return false;
return isValidFile(entry) || isValidDirectory(entry);
}
function isValidDirectory(entry) {
return (typeof entry === 'object' &&
entry &&
entry.type === ResolverStrategy.EntryKind.Directory &&
typeof entry.path === 'string' &&
entry.path &&
(typeof entry.files === 'undefined' ||
(Array.isArray(entry.files) && entry.files.every(isValidEntry))));
}
function isValidFile(entry) {
return (typeof entry === 'object' &&
entry &&
entry.type === ResolverStrategy.EntryKind.File &&
typeof entry.path === 'string' &&
entry.path);
}
function specToString(spec) {
return `${spec.spec}${spec.pathname}`;
}
class JSDelivrCdn {
constructor() {
this.name = 'jsdelivr';
this.specRx = /^\/((@[^/]+\/[^/@]+|[^/@]+)(?:@([^/]+))?)(.*)?$/;
}
isValidUrl(url) {
return url.scheme === JSDelivrCdn.protocol || url.authority === JSDelivrCdn.host;
}
normalizePackageListing(result) {
if (!result || typeof result !== 'object') {
throw new Error(`Unexpected package listing contents`);
}
const files = result.files;
if (!Array.isArray(files)) {
throw new Error(`Unexpected package listing contents`);
}
const mapChildEntry = (parent, child) => {
if (!child || typeof child !== 'object') {
throw new Error(`Unexpected entry in package listing contents`);
}
const name = child.name;
if (typeof name !== 'string') {
throw new Error(`Unexpected entry in package listing contents`);
}
const path = `${parent}/${name}`;
if (child.type === ResolverStrategy.EntryKind.Directory) {
const files = child.files;
if (!Array.isArray(files)) {
throw new Error(`Unexpected entry in package listing contents`);
}
return {
type: ResolverStrategy.EntryKind.Directory,
path,
files: files.map((file) => mapChildEntry(path, file)),
};
}
else if (child.type === ResolverStrategy.EntryKind.File) {
return {
type: ResolverStrategy.EntryKind.File,
path,
};
}
throw new Error(`Error mapping child entry in package file listing`);
};
return {
type: ResolverStrategy.EntryKind.Directory,
path: '/',
files: files.map((file) => mapChildEntry('', file)),
};
}
parseUrl(url) {
if (Uri.isUri(url)) {
url = url.path;
}
const prefix = `/npm`;
if (!url.startsWith(prefix)) {
throw new Error(`Unable to parse unexpected ${this.name} url: ${url}`);
}
url = url.slice(prefix.length);
/**
* 1: scope + name + version
* 2: scope + name
* 3: version?
* 4: pathname
*/
const matches = url.match(this.specRx);
if (!matches) {
throw new Error(`Unable to parse unexpected unpkg url: ${url}`);
}
return {
spec: matches[1],
name: matches[2],
version: matches[3] || '',
pathname: matches[4] || '',
};
}
urlForPackageFile(spec, pathname) {
return Uri.from({
scheme: JSDelivrCdn.protocol,
authority: JSDelivrCdn.host,
path: `/npm/${spec}${pathname}`,
});
}
urlForPackageList(spec) {
return Uri.from({
scheme: JSDelivrCdn.protocol,
authority: JSDelivrCdn.dataHost,
path: `/v1/package/npm/${spec}/tree`,
});
}
}
JSDelivrCdn.protocol = 'https';
JSDelivrCdn.host = 'cdn.jsdelivr.net';
JSDelivrCdn.dataHost = 'data.jsdelivr.com';
class UnpkgCdn {
constructor() {
this.name = 'unpkg';
this.UNPKG_SPEC_RX = /^\/((@[^/]+\/[^/@]+|[^/@]+)(?:@([^/]+))?)(.*)?$/;
}
isValidUrl(url) {
return url.scheme === UnpkgCdn.protocol || url.authority === UnpkgCdn.host;
}
normalizePackageListing(result) {
if (!isValidDirectory(result)) {
throw new Error(`Error normalizing directory listing`);
}
return result;
}
parseUrl(url) {
if (Uri.isUri(url)) {
url = url.path;
}
/**
* 1: scope + name + version
* 2: scope + name
* 3: version?
* 4: pathname
*/
const matches = url.match(this.UNPKG_SPEC_RX);
if (!matches) {
throw new Error(`Unable to parse unexpected unpkg url: ${url}`);
}
return {
spec: matches[1],
name: matches[2],
version: matches[3] || '',
pathname: matches[4] || '',
};
}
urlForPackageFile(spec, pathname) {
return Uri.from({
scheme: UnpkgCdn.protocol,
authority: UnpkgCdn.host,
path: `/${spec}${pathname}`,
});
}
urlForPackageList(spec) {
return Uri.from({
scheme: UnpkgCdn.protocol,
authority: UnpkgCdn.host,
path: `/${spec}/`,
query: 'meta',
});
}
}
UnpkgCdn.protocol = 'https';
UnpkgCdn.host = 'unpkg.com';
class CdnStrategy extends AbstractResolverStrategyWithRoot {
constructor(readUrlFn, cdn) {
super(cdn.urlForPackageFile('', ''));
this.contentCache = new Map();
this.locks = new Map();
this.packageEntriesCache = new Map();
this.packageJsonCache = new Map();
this.cdn = cdn;
this.readUrlFn = readUrlFn;
}
_withRootUriCheck(uri, fn) {
if (!Uri.isPrefixOf(this.rootUri, uri)) {
throw new Error(`This strategy is only able to handle URIs under '${this.rootUri.toString()}' and is unable to handle '${uri.toString()}'`);
}
return fn(this.rootUri);
}
async getUrlForBareModule(ctx, name, spec, path) {
const unresolvedUri = this.cdn.urlForPackageFile(`${name}@${spec}`, path);
const resolveReturn = await ctx.resolveUri(unresolvedUri);
return resolveReturn;
}
getCanonicalUrl(ctx, uri) {
return this._withRootUriCheck(uri, async () => {
const unresolvedSpec = this.cdn.parseUrl(uri);
const packageJsonReturn = ctx.runInChildContext('CdnStrategy._readPackageJsonWithCache', specToString(unresolvedSpec), (ctx) => this._readPackageJsonWithCache(ctx, unresolvedSpec));
const packageJson = isThenable(packageJsonReturn)
? await packageJsonReturn
: packageJsonReturn;
return {
uri: this.cdn.urlForPackageFile(`${packageJson.name}@${packageJson.version}`, unresolvedSpec.pathname),
};
});
// const results = all([ctx.getRootUrl(uri), ctx.getResolveRoot(uri)], ctx.token);
// const [rootUriResult, resolveRootResult] = isThenable(results) ? await results : results;
}
getResolveRoot(ctx, uri) {
return this._withRootUriCheck(uri, async () => {
const unresolvedSpec = this.cdn.parseUrl(uri);
const packageJsonReturn = this._readPackageJsonWithCache(ctx, unresolvedSpec);
const packageJson = isThenable(packageJsonReturn)
? await packageJsonReturn
: packageJsonReturn;
return {
uri: this.cdn.urlForPackageFile(`${packageJson.name}@${packageJson.version}`, '/'),
};
});
}
getRootUrl() {
return {
uri: this.cdn.urlForPackageFile('', ''),
};
}
listEntries(ctx, uri) {
return this._withRootUriCheck(uri, async () => {
const unresolvedSpec = this.cdn.parseUrl(uri);
const results = all([
ctx.getResolveRoot(uri),
this._readPackageJsonWithCache(ctx, unresolvedSpec),
this._readPackageEntriesWithCache(ctx, unresolvedSpec),
], ctx.token);
const [{ uri: resolveRootUri }, packageJson, entriesReturn] = isThenable(results)
? await results
: results;
const canonicalizedSpec = {
name: packageJson.name,
pathname: unresolvedSpec.pathname,
spec: `${packageJson.name}@${packageJson.version}`,
version: packageJson.version,
};
// Proactively cache the canonicalized package entries
this.packageEntriesCache.get(packageJson.name).set(packageJson.version, entriesReturn);
const traversalSegments = canonicalizedSpec.pathname.split('/').filter(Boolean);
let parentEntry = entriesReturn;
while (parentEntry && traversalSegments.length) {
const segment = traversalSegments.shift();
if (parentEntry.type !== ResolverStrategy.EntryKind.Directory || !parentEntry.files) {
throw new EntryNotFoundError(uri);
}
parentEntry = parentEntry.files.find((file) => file.type === ResolverStrategy.EntryKind.Directory && basename(file.path) === segment);
}
if (!parentEntry) {
throw new EntryNotFoundError(uri);
}
if (!parentEntry.files) {
return {
entries: [],
};
}
return {
entries: parentEntry.files.map((entry) => {
return {
type: entry.type,
uri: Uri.joinPath(resolveRootUri, `.${entry.path}`),
};
}),
};
});
}
readFileContent(ctx, uri) {
return this._withRootUriCheck(uri, () => {
const uriStr = uri.toString();
const cached = this.contentCache.get(uriStr);
if (cached === null) {
return Promise.reject(new EntryNotFoundError(uri));
}
if (cached) {
return cached;
}
ctx.recordVisit(uri, ResolverContext.VisitKind.File);
const readReturn = this.readUrlFn(uriStr, ctx.token);
if (readReturn === null) {
this.contentCache.set(uriStr, null);
return Promise.reject(new EntryNotFoundError(uri));
}
if (isThenable(readReturn)) {
const wrappedReturn = readReturn.then((data) => {
if (data === null) {
this.contentCache.delete(uriStr);
return Promise.reject(new EntryNotFoundError(uri));
}
const entry = { content: data };
this.contentCache.set(uriStr, entry);
return entry;
});
this.contentCache.set(uriStr, wrappedReturn);
return wrappedReturn;
}
const entry = { content: readReturn };
this.contentCache.set(uriStr, entry);
return entry;
});
}
_readPackageEntriesWithCache(ctx, spec) {
ctx.debug('%s._readPackageEntriesWithCache(%s)', this.constructor.name, specToString(spec));
return this._withLock(`packageEntries:${spec.name}`, () => {
let packageEntriesCacheForModule = this.packageEntriesCache.get(spec.name);
if (packageEntriesCacheForModule) {
const exactMatch = packageEntriesCacheForModule.get(spec.version);
if (exactMatch) {
// console.log('[HIT-EXACT] readPackageJsonWithCache(%s)', spec.spec);
return exactMatch;
}
const range = validRange(spec.version);
if (range) {
for (const [version, entries] of packageEntriesCacheForModule) {
if (satisfies(version, range)) {
return entries;
}
}
}
}
else {
packageEntriesCacheForModule = new Map();
this.packageEntriesCache.set(spec.name, packageEntriesCacheForModule);
}
return this._readPackageEntries(ctx, spec).then((rootDir) => {
packageEntriesCacheForModule.set(spec.version, rootDir);
return rootDir;
});
});
}
async _readPackageEntries(ctx, spec) {
ctx.debug('%s._readPackageEntries(%s)', this.constructor.name, specToString(spec));
const uri = this.cdn.urlForPackageList(spec.spec);
const href = uri.toString();
ctx.recordVisit(uri, ResolverContext.VisitKind.Directory);
const data = await checkCancellation(this.readUrlFn(href, ctx.token), ctx.token);
if (data === null) {
throw new EntryNotFoundError(spec);
}
const dataStr = ctx.decoder.decode(data);
return this.cdn.normalizePackageListing(JSON.parse(dataStr));
}
_readPackageJsonWithCache(ctx, spec) {
return this._withLock(`packageJson:${spec.name}`, () => {
let packageJsonCacheForModule = this.packageJsonCache.get(spec.name);
if (packageJsonCacheForModule) {
const exactMatch = packageJsonCacheForModule.get(spec.version);
if (exactMatch) {
// console.log('[HIT-EXACT] readPackageJsonWithCache(%s)', spec.spec);
for (const visit of exactMatch.visited) {
ctx.recordVisit(visit.uri, visit.type);
}
return exactMatch.packageJson;
}
const range = validRange(spec.version);
if (range) {
for (const [version, entry] of packageJsonCacheForModule) {
if (satisfies(version, range)) {
// console.log('[HIT] readPackageJsonWithCache(%s)', spec.spec);
for (const visit of entry.visited) {
ctx.recordVisit(visit.uri, visit.type);
}
return entry.packageJson;
}
}
}
}
else {
packageJsonCacheForModule = new Map();
this.packageJsonCache.set(spec.name, packageJsonCacheForModule);
}
return this._readPackageJson(spec, ctx).then((packageJson) => {
packageJsonCacheForModule.set(packageJson.version, { packageJson, visited: ctx.visited });
return packageJson;
});
});
}
async _readPackageJson(spec, ctx) {
ctx.debug('%s._readPackageJson(%s)', this.constructor.name, specToString(spec));
const uri = this.cdn.urlForPackageFile(spec.spec, '/package.json');
const contentReturn = ctx.readFileContent(uri);
const contentResult = isThenable(contentReturn) ? await contentReturn : contentReturn;
let manifest;
try {
manifest = parseBufferAsRootPackageJson(ctx.decoder, contentResult.content, spec.spec);
}
catch (err) {
throw new Error(`Error parsing manifest as json for package '${specToString(spec)}' at '${uri.toString()}': ${err.message}`);
}
// Since we know what the canonicalized version is now (we didn't until the promise resolved)
// and the package.json was parsed), we can proactively seed the content cache for the
// canonical url.
const canonicalHref = this.cdn
.urlForPackageFile(`${manifest.name}@${manifest.version}`, '/package.json')
.toString();
this.contentCache.set(canonicalHref, contentResult);
return manifest;
}
_withLock(lockKey, fn) {
const lock = this.locks.get(lockKey);
const runCriticalSection = () => {
const ret = fn();
if (isThenable(ret)) {
const locked = ret.then((result) => {
this.locks.delete(lockKey);
return result;
}, (err) => {
this.locks.delete(lockKey);
return Promise.reject(err);
});
this.locks.set(lockKey, locked);
return ret;
}
// No need to lock in non-promise
return ret;
};
if (isThenable(lock)) {
return lock.then(runCriticalSection);
}
return runCriticalSection();
}
static forJsDelivr(readUrlFn) {
return new CdnStrategy(readUrlFn, new JSDelivrCdn());
}
static forUnpkg(readUrlFn) {
return new CdnStrategy(readUrlFn, new UnpkgCdn());
}
}
const version = '0.56.2';
export { CdnStrategy, version };
//# sourceMappingURL=index.js.map