anytv-commons
Version:
Collection of libraries commonly used
905 lines (663 loc) • 19.7 kB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }
var cudl = _interopDefault(require('cuddle'));
var cld = _interopDefault(require('cld'));
class ExtendableError extends Error {
constructor (message) {
super(message);
this.name = this.constructor.name;
this.message = message;
if (typeof Error.captureStackTrace === 'function') {
Error.captureStackTrace(this, this.constructor);
}
else {
this.stack = (new Error(message)).stack;
}
}
toString () {
return this.stack;
}
}
class FatalError extends ExtendableError {
constructor (m) {
super(m);
}
}
class Config {
constructor (props) {
this.use_global = true;
this.prop = {};
if (props) {
this.set(props);
}
}
get (key) {
let val = this.prop[key];
if (!(key in this.prop) && this.use_global) {
val = this.constructor.GET(key);
}
return val;
}
set (key, value) {
if (typeof key === 'object' && typeof value === 'undefined') {
return this._set_prop(key);
}
this.prop[key] = value;
return this;
}
ignore_global () {
this.use_global = false;
return this;
}
_set_prop (prop) {
for (let key in prop) {
this.prop[key] = prop[key];
}
return this;
}
static get PROP () {
if (!this.prop) {
this.prop = {};
}
return this.prop;
}
static GET (key) {
return this.PROP[key];
}
static SET (key, value) {
if (typeof key === 'object' && typeof value === 'undefined') {
return this._SET_PROP(key);
}
if (key === 'throttle') {
cudl.throttle(value);
}
this.PROP[key] = value;
return this;
}
static _SET_PROP (prop) {
for (let key in prop) {
this.SET(key, prop[key]);
}
return this;
}
}
class Logger {
constructor () {
this.prefixes = Array.from(arguments);
this.levels = [
'silly',
'debug',
'verbose',
'info',
'warn',
'error'
];
this.levels.forEach(level =>
this[level] = this.log.bind(this, level)
);
}
prefix (_prefix) {
if (typeof _prefix === 'string') {
this.prefixes.push(_prefix);
}
else if (Array.isArray(_prefix)) {
this.prefixes = _prefix.concat(this.prefixes);
}
else {
throw new Error(`Invalid prefix "${_prefix}"`);
}
return this;
}
log (level) {
if (typeof level !== 'string') {
throw new Error(`Invalid log level "${level}"`);
}
if (!~this.levels.indexOf(level)) {
level = 'info';
}
const msg = [
this.prefixes.join('|'),
Array.prototype.map
.call(arguments, n =>
typeof n === 'string'
? n
: JSON.stringify(n)
)
.slice(1)
.join(' ')
];
if (this.constructor.custom_logger) {
this.constructor.custom_logger.log(level, msg.join(' '));
return this;
}
if (Config.GET('debug') || level === 'error') {
console.log([
new Date().toUTCString(),
level + ':\t'
]
.concat(msg)
.join(' '));
}
return this;
}
static USE (_logger) {
this.custom_logger = _logger;
return this;
}
}
class EventEmitter extends Logger {
constructor () {
super(...arguments);
this._listeners = {};
}
on (event, fn) {
if (typeof fn !== 'function') {
throw new Error(`Event listener for ${event} is not a function`);
}
if (!this._listeners[event]) {
this._listeners[event] = [];
}
this._listeners[event].push(fn);
this.debug(`listening to ${event}`);
return this;
}
emit (event, ...args) {
if (this._listeners[event]) {
this._listeners[event].forEach(fn => fn(...args));
this.silly(`emitted ${event} with `, ...args);
}
// if error event and no one is listening
else if (event === 'error') {
this.error(...args);
}
return this;
}
}
class Pipe extends EventEmitter {
constructor (name) {
super(...arguments);
this.name = name;
this._next = () => {};
this.config = {};
this._async_total = 0;
this._async_done = 0;
}
emit_error (err) {
this._done++;
this.emit('error', err);
}
emit_done (obj) {
this.debug('Done processing', obj);
this._done++;
this.emit('done', obj);
}
configure (cfg) {
this.config = cfg;
this.debug('Reconfigured');
return this;
}
receive () {
throw new FatalError('receive function was not overridden');
}
get _done () {
return this._async_done;
}
set _done (val) {
this._async_done = val;
this.emit('change', val, this._async_total);
}
get _total () {
return this._async_total;
}
set _total (val) {
this._async_total = val;
this.emit('change', this._async_done, val);
}
}
class Plumber extends Pipe {
constructor (config, args) {
super(...args, 'Plumber');
this.config = config;
this._prev = void 0;
this._source = void 0;
this.idle = 0;
this.pipes = [];
this.receive = this.receive.bind(this);
this.emit_error = this.emit_error.bind(this);
this.emit_change = this.emit_change.bind(this);
this.toString = this.toString.bind(this);
}
source (src) {
this.debug(`Source: ${src.constructor.name}`);
this.pipes.push(src);
src.configure(this.config)
.prefix(this.prefixes);
this._source = src;
this._prev = src;
return this;
}
connect (pipe) {
this.debug(`Connecting ${pipe.constructor.name} to ${this._prev.constructor.name}`);
this._prev.on('done', pipe.receive);
this._prev = pipe;
if (this._prev !== this) {
this.pipes.push(this._prev);
this._prev
.configure(this.config)
.prefix(this.prefixes)
.on('error', this.emit_error(this._prev))
.on('change', this.emit_change(this._prev));
}
return this;
}
blast (cb) {
this.next = cb;
this.debug('Source is now starting');
this.connect(this)
._source
.start();
this.idler = setInterval(() => {
this.idle++;
if (this.idle >= (this.config.get('idle_timeout') || 60)) {
this.info('Idled');
this._exit();
}
}, 1000);
return this;
}
// @@override
receive (channels) {
this.emit('channels', channels);
return this;
}
// @@override
emit_error (pipe) {
return err => {
this.emit('error', pipe.name, err);
if (err instanceof FatalError) {
this._exit();
}
};
}
emit_change (pipe) {
return function () {
this.idle = 0;
this.emit('change', pipe.name, ...arguments);
if (this._is_all_done()) {
this._exit();
}
}.bind(this);
}
// @@override
toString () {
return '\n' + this.pipes
.map(p => `${p.name} done with ${p._done} of ${p._total}`)
.concat(`Idle for ${this.idle} seconds`)
.join('\n');
}
_exit () {
clearInterval(this.idler);
this.next();
return this;
}
_is_all_done () {
return !this.pipes
.filter(pipe => pipe._total === 0 || pipe._done !== pipe._total)
.length;
}
}
/**
* Use cases
* You can use it in any YouTube Data API
* Functions
* Change api_key or access token when a limit is hit
* Definitions
* Token - an API key or an access token
* Config keys
* auth_key api_key | access_token
* change_key function that accepted a callback
* change_token
*/
class YTRequest extends Logger {
constructor (prefixes, method, url) {
super(...prefixes, 'YTRequest');
this.done = false;
this.request = cudl.request(method).to(url);
this._additional_args = [];
// default auth type
this.auth_type = 'auto';
this.then = this.then.bind(this);
this._authorize = this._authorize.bind(this);
this._catch_errors = this._catch_errors.bind(this);
this._handle_error = this._handle_error.bind(this);
this._can_change_auth = this._can_change_auth.bind(this);
this._get_new_token = this._get_new_token.bind(this);
this._change_token = this._change_token.bind(this);
}
configure (cfg) {
this.config = cfg;
this.debug('Done setting configuration');
return this;
}
set_auth_type (type = 'api_key') {
this.auth_type = type;
return this;
}
args () {
this._additional_args = Array.from(arguments);
this.request.args(...arguments);
return this;
}
send (payload) {
this.request.send(payload);
return this;
}
/**
* @error cudl: when YouTube returns a non-200 status code
* @error cudl: when there's a network problem
* @error cudl: when we hit max retries
* @error cudl: when JSON parse fails
* @error when api key or access token is missing
* @error when there's an error in getting new access token
* @error when there's an error in getting new api key
*/
then (cb) {
this.debug('calling then');
this.next = cb;
this._authorize()
.request
.then(this._catch_errors);
setTimeout(() => {
if (!this.done) {
this.done = true;
this.next(
new Error('YTRequest timed out'),
null,
this.request,
this._additional_args
);
}
}, 1000 * 60 * 5);
}
_authorize () {
if (this.auth_type === 'auto') {
this.auth_type = this.config.get('access_token')
? 'access_token'
: 'api_key';
}
const token = this.config.get(this.auth_type);
if (!token) {
this.done = true;
this.next(
new FatalError('Need an api key or access token'),
null,
this.request,
this._additional_args
);
return this;
}
this.silly('token', token);
if (this.auth_type === 'access_token') {
this.request.set_header('Authorization', 'Bearer ' + token);
}
else {
this.request.data.key = token;
}
this.debug('Done authorizing the request');
return this;
}
_catch_errors (err) {
this.verbose('Request result received');
if (err && this._handle_error(err)) {
return;
}
if (!this.done) {
this.done = true;
this.next(...arguments);
}
}
_handle_error (err) {
const request = this.request;
// if YouTube server error, simply calm down for 5 seconds
if (err.code >= 500) {
request.errors.push(err);
setTimeout(request.retry.bind(request), 5000);
this.debug('Error 500, sleeping for a while');
return true;
}
let reason = (err.response
&& err.response.error
&& err.response.error.errors
&& err.response.error.errors[0]
&& err.response.error.errors[0].reason)
|| '';
if (err.code === 403
&& ~reason.indexOf('Exceeded')
&& this._can_change_auth())
{
request.errors.push(err);
this._get_new_token(reason);
this.debug('Error 403', reason, 'changing auth');
return true;
}
return false;
}
_can_change_auth () {
return typeof this.config.get('change_auth') === 'function';
}
_get_new_token (reason) {
this.debug('Changing key');
if (this.constructor.CHANGING) {
return;
}
this.constructor.CHANGING = true;
this.config.get('change_auth')(
this.auth_type,
this.config.get(this.auth_type),
reason,
this._change_token
);
}
_change_token (err, new_token) {
const request = this.request;
if (err) {
return this.error('Error in getting new token', err)
.next(new FatalError('OUT_OF_TOKENS'));
}
this.config.set(this.auth_type, new_token);
this._authorize();
request.retry();
}
static get CHANGING () {
return this.changing || false;
}
static set CHANGING (val) {
this.changing = val;
}
}
class Grouper extends Pipe {
constructor (max, seconds = 10) {
super('Grouper');
this._max = max;
this._stack = [];
this._seconds = seconds;
this._timer = 0;
this.receive = this.receive.bind(this);
this._regulate = this._regulate.bind(this);
setInterval(this._regulate, 1000);
}
// @@override
receive (channels) {
this._stack = this._stack.concat(channels);
while (this._stack.length >= this._max) {
this._timer = this._seconds;
this._regulate();
}
return this;
}
_regulate () {
this._timer++;
if (this._timer >= this._seconds && this._stack.length) {
this._timer = 0;
this._total++;
this.emit_done(this._stack.splice(0, this._max));
}
}
}
function pad (num, size) {
return ('000000000' + num).substr(-(size || 2));
}
/**
* Detects language of the item's description or title
* Attaches `written_language` on the item
*/
class CLD extends Pipe {
constructor () {
super('CLD');
this.receive = this.receive.bind(this);
this._extract_language = this._extract_language.bind(this);
}
// @@override
receive (items) {
this.debug('Received', items);
for (let i = 0; i < items.length; i += 1) {
const item = items[i];
if (item.flag
&& this._is_done_for_today(item)) {
continue;
}
items.splice(i, 1);
// decrement index because the array length was also decremented
i -= 1;
this._total++;
this._detect(item);
}
if (items.length) {
this._total++;
this.debug(`passing ${items.length}`);
this.emit_done(items);
}
}
_is_done_for_today (item) {
return item.flag
? pad(item.flag, 10)[5] === '1'
: false;
}
_detect (item) {
this.debug(`Extracting language of ${item.item_id}`);
cld.detect(item.description || item.title, this._extract_language(item));
}
_extract_language (item) {
return (err, lang) => {
item.written_language = lang
&& lang.languages
&& lang.languages.length
&& lang.languages[0].name
? lang.languages[0].name.toLowerCase()
: '';
this.debug(`Done extracting language of ${item.item_id}; ${item.written_language}`);
this.emit_done([item]);
};
}
}
function get_page_token (index) {
const alphanum = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
const i = index;
let first_shifter = 0;
let second_shifter = 0;
/**
* FIRST
* > 0 ABCDEFGHIJKLMNOP
* > 256 IJKLMNOP
*/
let first_idx = ~~(i / 16);
if (i >= Math.pow(2, 8)) {
first_idx = first_idx % 8;
first_shifter = 8;
}
/**
* SECOND
* let a = 2^13
* > 0 AEIMQUYcgkosw048 shift 0
* > a BFJNRVZdhlptx159 shift 1
* ~~(i/a) a is even CGKOSWaeimquy260 shift 2
* ~~(i/a) is odd DHLPTXbfjnrvz37- shift 3
*/
let second_idx = i % 16;
if (i > Math.pow(2, 13)) {
second_shifter = 1;
if (i > (Math.pow(2, 13) * 2)) {
second_shifter += (~~(i/Math.pow(2, 13)) % 2) ? 2 : 1;
}
}
/**
* THIRD
* @todo learn the pattern for the suffix
* ABCDEFGHIJKLMNOP
* QRSTUVWXYZabcdef
* ghijklmnopqrstuv
* wxyz01234567890-
* let a = (2^7) = 128
* > 0 0 Q = index 16
* > a 128 [alphanum] + E = index 4
* > a^2 16384 [alphanum] + AR = index 1 18
* > a^2 * 2 32768 [alphanum] + Ah = index 1 34 34-18 = 16
* > a^2 * 3 49152 [alphanum] + Ax = index 1 50 50-34 = 16
* > a^2 * 4 65536 [alphanum] + BB = index 2 2 66-50 = 16
* > a^2 * 5 81920 [alphanum] + BR
* > a^2 * 6 98304 [alphanum] + ?? might be Bh
* > a^2 * 7 114688 [alphanum] + ?? might be Bx
* > a^2 * 8 131072 [alphanum] + ?? might be CB
*/
let last_idx = ~~(i / Math.pow(2, 7));
let last = alphanum[last_idx % alphanum.length];
let last_suffix;
if (!last_idx) {
last = '';
last_suffix = 'Q';
}
if (i >= 128) {
last_suffix = 'E';
}
if (i >= 16384) {
last_suffix = 'AR';
}
if (i >= 32768) {
last_suffix = 'Ah';
}
if (i >= 49152) {
last_suffix = 'Ax';
}
if (i >= 65535) {
last_suffix = 'BB';
}
if (i >= 81920) {
last_suffix = 'BR';
}
if (i >= 98304) {
throw new Error(`PageToken last suffix unknown for >= 98304.
Figure out the last suffix by inspecting nextPageToken of page ${get_page_token(98303)} with maxResults: 1`);
}
last += last_suffix;
// build token
// C and AA looks constant
return [
'C',
alphanum[first_idx + first_shifter],
alphanum[(((second_idx % 16) * 4) + second_shifter) % alphanum.length],
last,
'AA'
].join('');
}
exports.ExtendableError = ExtendableError;
exports.FatalError = FatalError;
exports.Config = Config;
exports.Plumber = Plumber;
exports.YTRequest = YTRequest;
exports.EventEmitter = EventEmitter;
exports.Logger = Logger;
exports.Pipe = Pipe;
exports.Grouper = Grouper;
exports.CLD = CLD;
exports.PageToken = get_page_token;