@rubic/catalog-fetcher
Version:
Rubic Catalog fetcher
326 lines • 12.1 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
///<reference path="../lib/catalog.d.ts" />
const GitHub = require("@octokit/rest");
const Ajv = require("ajv");
const CJSON = require("comment-json");
const fs = require("fs");
const path = require("path");
const request = require("request");
const pify = require("pify");
const decompress = require("decompress");
exports.CATALOG_JSON = "catalog.json";
exports.JSON_ENCODING = "utf8";
exports.REPOSITORY_JSON = "rubic-repository.json";
exports.RELEASE_JSON = "release.json";
exports.ASSET_PATTERN = /\.(zip|tar\.gz|tgz)$/i;
exports.OFFICIAL_CATALOG = {
owner: "kimushu",
repo: "rubic-catalog",
branch: "vscode-master"
};
const USER_AGENT = "@rubic/catalog-fetcher";
let ajv;
function loadSchema() {
ajv = new Ajv();
let schema = JSON.parse(fs.readFileSync(path.join(__dirname, "catalog.schema.json"), exports.JSON_ENCODING));
ajv.compile(schema);
}
loadSchema();
/**
* Get ajv-based validator for Rubic catalog schema
* @param name Name of interface
*/
function getValidator(name) {
if (name == null) {
return ajv.getSchema(undefined);
}
return ajv.getSchema(`#/definitions/RubicCatalog.${name}`);
}
exports.getValidator = getValidator;
/**
* Catalog fetcher for Rubic
*/
class RubicCatalogFetcher {
/**
* Construct Rubic catalog fetcher
* @param options Options for fetcher
*/
constructor(options = {}) {
this.options = options;
this.limit_used = 0;
this.limit_remaining = null;
this.temp_id_next = 1;
const ghopt = {
headers: {
"user-agent": USER_AGENT + (options.userAgent ? ` (${options.userAgent})` : "")
},
};
if (options.proxy != null) {
ghopt.proxy = options.proxy;
}
if (options.rejectUnauthorized != null) {
ghopt.rejectUnauthorized = options.rejectUnauthorized;
}
this.gh = new GitHub(ghopt);
this.logger = options.logger;
if (this.logger == null) {
this.logger = {};
}
if (this.logger.log == null) {
this.logger.log = (msg, ...args) => console.log(msg, ...args);
}
if (this.logger.info == null) {
this.logger.info = (...args) => this.logger.log("[INFO]", ...args);
}
if (this.logger.warn == null) {
this.logger.warn = (...args) => this.logger.log("[WARN]", ...args);
}
if (this.logger.error == null) {
this.logger.error = (...args) => this.logger.log("[ERROR]", ...args);
}
if (options.auth) {
this.gh.authenticate(options.auth);
}
}
/**
* Fetch repository data
* @param repo Repository info
* @param current Current data to merge
* @param ver Version suffix
*/
fetchRepository(repo, current, ver) {
let { branch } = repo;
if (branch == null) {
branch = "master";
}
this.logger.log(`Fetching repository: ${repo.host}:${repo.owner}/${repo.repo}#${branch}${current != null ? " (Merge mode)" : ""}`);
if (current == null) {
current = {};
}
if (current.uuid == null) {
current.uuid = `00000000-${("0000" + (this.temp_id_next++).toString()).substr(-4)}-0000-0000-000000000000`;
this.logger.info(`Assigned a temporary UUID for this repository (${current.uuid})`);
}
current.host = repo.host;
current.owner = repo.owner;
current.repo = repo.repo;
current.branch = branch;
return this.fetchRepositoryJSON(current, ver).then(() => {
return current;
});
}
/**
* Validate object and report
* @param obj Object to validate
* @param name Name of interface
* @param title Name of object
*/
validate(obj, name, title) {
const validator = getValidator(name);
if (!validator(obj)) {
let { errors } = validator;
this.logger.error(`${title} has ${errors.length} error(s)`);
for (let i = 0; i < errors.length; ++i) {
let e = errors[i];
this.logger.error(`- (${e.keyword}) ${e.dataPath}: ${e.message}`);
}
throw new Error(`invalid ${title}`);
}
}
/**
* Fetch REPOSITORY_JSON
* @param repo Object to store fetched data
* @param ver Version suffix
*/
fetchRepositoryJSON(repo, ver) {
return Promise.resolve()
.then(() => {
// Fetch REPOSITORY_JSON
return this.recordRateLimit(this.gh.repos.getContent({
owner: repo.owner,
repo: repo.repo,
ref: repo.branch,
path: exports.REPOSITORY_JSON
}));
})
.then((result) => {
// Validate REPOSITORY_JSON
const repos_json = CJSON.parse(Buffer.from(result.data.content, result.data.encoding).toString(), null, true);
this.validate(repos_json, `RepositoryDetail${ver}`, exports.RELEASE_JSON);
if (repos_json.releases == null) {
repos_json.releases = repo.cache && repo.cache.releases;
}
repo.cache = repos_json;
return this.fetchReleaseList(repo, ver);
});
}
/**
* Fetch releases
* @param repo Object to store fetched data
* @param ver Version suffix
*/
fetchReleaseList(repo, ver) {
return Promise.resolve()
.then(() => {
// Fetch release list
return this.recordRateLimit(this.gh.repos.getReleases({
owner: repo.owner,
repo: repo.repo
}));
})
.then((result) => {
// Update releases array
let dest = repo.cache.releases;
let merge = (dest != null);
let tags_src = [];
let tags_exists = [];
let tags_updated = [];
if (merge) {
tags_src = dest.map((rel) => rel.tag);
}
else {
dest = repo.cache.releases = [];
}
result.data.forEach((rel) => {
if (rel.draft) {
return; // Do not include draft releases
}
let url;
let updated_at;
(rel.assets || []).forEach((asset) => {
if (!url && exports.ASSET_PATTERN.test(asset.name)) {
url = asset.browser_download_url;
updated_at = Date.parse(asset.updated_at);
}
});
let tag = rel.tag_name;
if (url == null) {
this.logger.warn(`No asset data for tag '${tag}'. Skipping`);
return;
}
let index = tags_src.indexOf(tag);
let old_value;
if (index < 0) {
// New item
index = dest.length;
}
else {
// Replace item
old_value = dest[index];
}
dest[index] = {
tag,
name: rel.name,
description: rel.body,
published_at: Date.parse(rel.published_at),
updated_at,
author: rel.author.login,
url,
cache: (old_value && old_value.cache)
};
tags_exists.push(tag);
if (old_value == null || updated_at > old_value.updated_at) {
tags_updated.push(tag);
if (old_value != null) {
this.logger.info(`Found an updated release: ${tag}`);
}
}
// Report new release
if (!merge || tags_src.indexOf(tag) < 0) {
this.logger.info(`Found a new release: ${tag}`);
}
});
// Sort from newest to oldest
dest.sort((a, b) => b.published_at - a.published_at);
// Report removed releases
for (let tag of tags_src) {
if (tags_exists.indexOf(tag) < 0) {
dest.splice(dest.findIndex((rel) => rel.tag === tag), 1);
this.logger.info(`Removed an old release: ${tag}`);
}
}
// Fetch asset data
return tags_updated.reduce((promise, tag) => {
return promise
.then(() => {
let rel = dest.find((rel) => rel.tag === tag);
return this.fetchReleaseDetail(rel, ver);
});
}, Promise.resolve());
});
}
/**
* Fetch ReleaseDetail
* @param rel Object to store fetched data
* @param ver Version suffix
*/
fetchReleaseDetail(rel, ver) {
this.logger.info(`Downloading asset data: ${rel.tag} (${rel.url})`);
let rqopt = { url: rel.url, encoding: null };
if (this.options.proxy != null) {
rqopt.proxy = this.options.proxy;
}
if (this.options.rejectUnauthorized != null) {
rqopt.strictSSL = this.options.rejectUnauthorized;
}
return pify(request)(rqopt)
.then((response) => {
this.logger.info(`Decompressing asset data: ${rel.tag}`);
return decompress(response.body);
})
.then((files) => {
// Find RELEASE_JSON
let rel_json_file = files.find((file) => file.path === exports.RELEASE_JSON);
if (rel_json_file == null) {
throw new Error(`${exports.RELEASE_JSON} not found in asset data`);
}
// Validate RELEASE_JSON
let rel_json = CJSON.parse(rel_json_file.data.toString(), null, true);
this.validate(rel_json, `ReleaseDetail${ver}`, exports.RELEASE_JSON);
// Check files for all variations
for (let v of rel_json.variations) {
let file = files.find((file) => file.path === v.path);
if (file == null) {
throw new Error(`Variation file not found: ${v.path}`);
}
// Check template files for all runtimes
for (let r of v.runtimes) {
if (r.template != null) {
let file = files.find((file) => file.path === r.template);
if (file == null) {
throw new Error(`Template file not found: ${r.template}`);
}
}
}
}
// Store data
rel.cache = rel_json;
});
}
/**
* RateLimit recording for GitHub API
* @param data GitHub API result
*/
recordRateLimit(data) {
return Promise.resolve(data)
.then((result) => {
const headers = (result && result.headers);
if (headers) {
++this.limit_used;
this.limit_remaining = parseInt(headers["x-ratelimit-remaining"]);
}
return result;
});
}
/**
* Report RateLimit status for GitHub API
*/
reportRateLimit() {
if (this.limit_remaining != null) {
this.logger.info(`(GitHub API RateLimit) Used: ${this.limit_used}, Remaining: ${this.limit_remaining}`);
}
}
}
exports.RubicCatalogFetcher = RubicCatalogFetcher;
//# sourceMappingURL=index.js.map