@wabarc/broker
Version:
Pack and store webpages from Telegram channel.
346 lines • 17 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DutyMachine = void 0;
var rest_1 = require("@octokit/rest");
var utils_1 = require("./utils");
var axios_1 = __importDefault(require("axios"));
var DutyMachine = /** @class */ (function () {
function DutyMachine(contract) {
this.contract = contract;
this.prefix = 'broker.dtmc.';
var token = contract.token, owner = contract.owner, repo = contract.repo, endpoint = contract.endpoint;
this.endpoint = endpoint || 'https://archives.duty-machine.now.sh/api/submit';
this.contract = contract;
this.credentials = { owner: owner, repo: repo };
try {
this.octokit = new rest_1.Octokit({ auth: token });
}
catch (_) {
throw new Error('Bad credentials.');
}
}
DutyMachine.prototype.process = function (stages) {
return __awaiter(this, void 0, void 0, function () {
var pushed, tasks, _i, tasks_1, task, succeed, _a, _b, uri;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
if (!stages || stages.length < 1) {
return [2 /*return*/, false];
}
pushed = [];
tasks = this.transform(stages);
_i = 0, tasks_1 = tasks;
_c.label = 1;
case 1:
if (!(_i < tasks_1.length)) return [3 /*break*/, 8];
task = tasks_1[_i];
if (!Object.prototype.hasOwnProperty.call(task, 'uris')) {
return [3 /*break*/, 7];
}
if (task.uris.length === 0) {
return [3 /*break*/, 7];
}
succeed = false;
_a = 0, _b = task.uris;
_c.label = 2;
case 2:
if (!(_a < _b.length)) return [3 /*break*/, 5];
uri = _b[_a];
return [4 /*yield*/, this.submit(uri)];
case 3:
if (_c.sent()) {
succeed = true;
}
_c.label = 4;
case 4:
_a++;
return [3 /*break*/, 2];
case 5:
if (succeed) {
pushed.push(task);
}
return [4 /*yield*/, utils_1.sleep(500)];
case 6:
_c.sent();
_c.label = 7;
case 7:
_i++;
return [3 /*break*/, 1];
case 8:
if (!(pushed.length === 0)) return [3 /*break*/, 10];
return [4 /*yield*/, this.submit('')];
case 9:
_c.sent();
_c.label = 10;
case 10: return [4 /*yield*/, this.tagging(pushed)];
case 11: return [2 /*return*/, _c.sent()];
}
});
});
};
DutyMachine.prototype.latestID = function () {
return __awaiter(this, void 0, void 0, function () {
var matchTag, response, tag, latest, id, _1;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
matchTag = function (tags) {
var regexp = new RegExp((_this.prefix + "\\d+\\-\\d+").replace(/\./g, '\\$&'), 'g');
for (var _i = 0, _a = Object.values(tags); _i < _a.length; _i++) {
var tag = _a[_i];
if (regexp.test(tag.name)) {
return tag;
}
}
};
_a.label = 1;
case 1:
_a.trys.push([1, 3, , 4]);
return [4 /*yield*/, this.octokit.repos.listTags(this.credentials)];
case 2:
response = _a.sent();
if (!response || !response.data) {
return [2 /*return*/, 0];
}
tag = matchTag(response.data);
if (tag === undefined) {
return [2 /*return*/, 0];
}
latest = tag.name || '';
id = latest.replace(this.prefix, '').split('-')[1] || '';
return [2 /*return*/, id.length > 0 ? parseInt(id) : 0];
case 3:
_1 = _a.sent();
return [2 /*return*/, 0];
case 4: return [2 /*return*/];
}
});
});
};
DutyMachine.prototype.tagging = function (stages) {
return __awaiter(this, void 0, void 0, function () {
var from, to, commit, credentials, tagSha, response, err_1, ref, _2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
console.info('Process tagging start...');
if (!stages || stages.length < 1) {
console.info('Process tagging failure, message: without task, skip.');
return [2 /*return*/, false];
}
from = stages[0].id || -1;
to = stages[stages.length - 1].id || -1;
if (from <= 0 || to <= 0 || from > to) {
console.info('Process tagging failure, message: params [from, to] invalid.');
return [2 /*return*/, false];
}
commit = 'commit';
credentials = {
owner: this.credentials.owner,
repo: this.credentials.repo,
tag: "" + this.prefix + from + "-" + to,
message: "\n" + JSON.stringify(stages, null, 2) + "\n",
object: this.credentials.sha,
type: commit,
};
_a.label = 1;
case 1:
_a.trys.push([1, 3, , 4]);
return [4 /*yield*/, this.octokit.git.createTag(credentials)];
case 2:
response = _a.sent();
tagSha = response.data.sha || this.credentials.sha;
return [3 /*break*/, 4];
case 3:
err_1 = _a.sent();
console.warn("Process tagging failure, message: create tag object error, details: " + err_1.message + ", data: " + JSON.stringify(credentials));
return [2 /*return*/, false];
case 4:
ref = {
owner: this.credentials.owner,
repo: this.credentials.repo,
ref: "refs/tags/" + credentials.tag,
sha: tagSha,
};
_a.label = 5;
case 5:
_a.trys.push([5, 7, , 8]);
// doc: https://octokit.github.io/rest.js/v18#git-create-ref
return [4 /*yield*/, this.octokit.git.createRef(ref)];
case 6:
// doc: https://octokit.github.io/rest.js/v18#git-create-ref
_a.sent();
return [3 /*break*/, 8];
case 7:
_2 = _a.sent();
return [2 /*return*/, false];
case 8:
console.info('done.');
return [2 /*return*/, true];
}
});
});
};
DutyMachine.prototype.transform = function (stages) {
var _this = this;
var result = [];
if (stages.length === 0) {
return result;
}
var pickup = function (uris) {
return Object.assign(uris).filter(function (uri) { return _this.allow(uri); });
};
Object.assign(stages).map(function (item) {
var id = item.id, stage = item.stage;
var uris = [];
var existOrig = Object.prototype.hasOwnProperty.call(stage, 'orig');
var existPH = Object.prototype.hasOwnProperty.call(stage, 'ph');
if (existOrig && existPH) {
if (stage.ph.length > stage.orig.length) {
uris.push.apply(uris, stage.ph);
}
else {
var orig = pickup(stage.orig);
var ph = pickup(stage.ph);
var p = orig.length > 0 ? orig : ph;
uris.push.apply(uris, p);
}
}
else if (existOrig) {
uris.push.apply(uris, pickup(stage.orig));
}
else if (existPH) {
// Assign Telegraph URI if original URI no matched
uris.push.apply(uris, pickup(stage.ph));
}
result.push({ id: id, uris: uris });
});
return result;
};
DutyMachine.prototype.allow = function (url) {
var allowList = [
/https?:\/\/mp\.weixin\.qq\.com/,
/https?:\/\/matters\.news/,
/https?:\/\/chinadigitaltimes\.net/,
/https?:\/\/www\.rfa\.org/,
/https?:\/\/telegra\.ph/,
/https?:\/\/(www|zhuanlan)\.zhihu\.com\/(question\/\d+\/answer\/\d+|p\/\d+)/,
/https?:\/\/(www|m)\.douban\.com\/(note|people|doubanapp\/dispatch\?uri=\/(note\/|status\/\d+|group\/topic\/)|group\/topic\/)/,
/https?:\/\/(www\.|m\.|card\.|weibointl\.api\.)?weibo\.(com|cn)\/(status\/\w+|\d+\/|share\/\d+|detail\/\d+|ttarticle\/p\/show|article\/m\/show\/id)/,
/https?:\/\/shimo\.im\/docs\/\w+/,
// /https?:\/\/web\.archive\.org\/web\/\d+\/\S+/,
/https?:\/\/(www)\.acfun\.cn\/a\/\w+/,
/https?:\/\/(www)\.bilibili\.com\/read\/\w+/,
/https?:\/\/archiveofourown\.org\/works\/\w+/,
];
// return matched url.
var matched = allowList.filter(function (regexp) {
return regexp.test(url);
});
return matched.length > 0;
};
DutyMachine.prototype.submit = function (url) {
return __awaiter(this, void 0, void 0, function () {
var params, response, err_2, ext, response, succeed, _3;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
console.info('Process submit start... ');
if (url && url.length > 0) {
params = new URLSearchParams();
params.append('url', url);
axios_1.default.post(this.endpoint, params, { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } });
}
this.credentials.path = "foo.bar";
_a.label = 1;
case 1:
_a.trys.push([1, 3, , 4]);
return [4 /*yield*/, this.octokit.repos.getContent({
owner: this.credentials.owner,
repo: this.credentials.repo,
path: this.credentials.path,
})];
case 2:
response = _a.sent();
this.credentials.sha = response.data['sha'];
return [3 /*break*/, 4];
case 3:
err_2 = _a.sent();
if (err_2.status &&
err_2.status === 403 &&
err_2.errors &&
err_2.errors[0]['code'] &&
err_2.errors[0]['code'] === 'too_large') {
ext = Math.random().toString(36).substring(2, 7) + '.html';
this.credentials.path = this.credentials.path.replace(/\.(htm|html)$/g, ext);
}
return [3 /*break*/, 4];
case 4:
this.credentials.message = url ? "Submit " + url + " to duty-machine" : "Ignore";
this.credentials.content = Buffer.from(Math.random().toString(36)).toString('base64');
_a.label = 5;
case 5:
_a.trys.push([5, 7, , 8]);
return [4 /*yield*/, this.octokit.repos.createOrUpdateFileContents(this.credentials)];
case 6:
response = _a.sent();
succeed = response.data !== undefined && response.data.commit !== undefined;
if (succeed) {
this.credentials.sha = response.data.commit.sha || '';
}
return [2 /*return*/, succeed];
case 7:
_3 = _a.sent();
console.log(_3);
console.info('Process createContent failure...');
return [2 /*return*/, false];
case 8: return [2 /*return*/];
}
});
});
};
return DutyMachine;
}());
exports.DutyMachine = DutyMachine;
//# sourceMappingURL=duty-machine.js.map