smc-hub
Version:
CoCalc: Backend webserver component
481 lines • 23.6 kB
JavaScript
"use strict";
/*
* This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
* License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __values = (this && this.__values) || function(o) {
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
if (m) return m.call(o);
if (o && typeof o.length === "number") return {
next: function () {
if (o && i >= o.length) o = void 0;
return { value: o && o[i++], done: !o };
}
};
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.calc_stats = void 0;
var async_utils_1 = require("smc-util/async-utils");
var schema_1 = require("smc-util/schema");
var misc = __importStar(require("smc-util/misc"));
var defaults = misc.defaults;
var required = defaults.required;
var _ = require("underscore");
var all_results = require("../postgres-base").all_results;
// some stats queries have to crunch a lot of rows, which could take a bit
// we give them a couple of minutes each…
var QUERY_TIMEOUT_S = 300;
var _stats_cached = null;
var _stats_cached_db_query = null;
function _count_timespan(db, opts) {
var _a, _b;
return __awaiter(this, void 0, void 0, function () {
var table, field, age_m, upper_m, where, result;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
opts = defaults(opts, {
table: required,
field: undefined,
age_m: undefined,
upper_m: undefined, // defaults to zero minutes (i.e. "now")
});
table = opts.table, field = opts.field, age_m = opts.age_m, upper_m = opts.upper_m;
where = {};
if (field != null) {
if (age_m != null) {
where[field + " >= $::TIMESTAMP"] = misc.minutes_ago(age_m);
}
if (upper_m != null) {
where[field + " <= $::TIMESTAMP"] = misc.minutes_ago(upper_m);
}
}
return [4 /*yield*/, async_utils_1.callback2(db._query, {
query: "SELECT COUNT(*) FROM " + table,
where: where,
timeout_s: QUERY_TIMEOUT_S,
})];
case 1:
result = _c.sent();
// count_result
return [2 /*return*/, parseInt((_b = (_a = result === null || result === void 0 ? void 0 : result.rows) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.count)];
}
});
});
}
function _count_opened_files(db, opts) {
return __awaiter(this, void 0, void 0, function () {
var age_m, key, data, distinct, q, res, rows, values;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
opts = defaults(opts, {
age_m: undefined,
key: required,
data: required,
distinct: required, // true or false
});
age_m = opts.age_m, key = opts.key, data = opts.data, distinct = opts.distinct;
q = "WITH filenames AS (\n SELECT " + (distinct ? "DISTINCT" : "") + " event ->> 'filename' AS fn\n FROM project_log\n WHERE time BETWEEN $1::TIMESTAMP AND NOW()\n AND event @> '{\"action\" : \"open\"}'::jsonb\n), ext_count AS (\n SELECT COUNT(*) as cnt, lower(reverse(split_part(reverse(fn), '.', 1))) AS ext\n FROM filenames\n GROUP BY ext\n)\nSELECT ext, cnt\nFROM ext_count\nWHERE ext IN ('sagews', 'ipynb', 'tex', 'rtex', 'rnw', 'x11',\n 'rmd', 'txt', 'py', 'md', 'sage', 'term', 'rst', 'lean',\n 'png', 'svg', 'jpeg', 'jpg', 'pdf',\n 'tasks', 'course', 'sage-chat', 'chat')\nORDER BY ext\n";
return [4 /*yield*/, async_utils_1.callback2(db._query, {
query: q,
params: [misc.minutes_ago(age_m)],
timeout_s: QUERY_TIMEOUT_S,
})];
case 1:
res = _a.sent();
rows = res.rows.map(function (x) { return misc.copy(x); });
values = _.object(_.pluck(rows, "ext"), _.pluck(rows, "cnt"));
data[key] = values;
return [2 /*return*/];
}
});
});
}
function check_local_cache(_a) {
var update = _a.update, ttl_dt = _a.ttl_dt, ttl = _a.ttl, ttl_db = _a.ttl_db, dbg = _a.dbg;
if (_stats_cached == null)
return null;
// decide if cache should be used -- tighten interval if we are allowed to update
var offset_dt = update ? ttl_dt : 0;
var is_cache_recent = _stats_cached.time > misc.seconds_ago(ttl - offset_dt);
// in case we aren't allowed to update and the cache is outdated, do not query db too often
var did_query_recently = _stats_cached_db_query != null &&
_stats_cached_db_query > misc.seconds_ago(ttl_db);
if (is_cache_recent || did_query_recently) {
dbg("using locally cached stats from " + (new Date().getTime() - _stats_cached.time) / 1000 + " secs ago.");
return _stats_cached;
}
return null;
}
function check_db_cache(_a) {
var _b;
var db = _a.db, update = _a.update, ttl = _a.ttl, ttl_dt = _a.ttl_dt, dbg = _a.dbg;
return __awaiter(this, void 0, void 0, function () {
var res, x, err_1;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
_c.trys.push([0, 2, , 3]);
return [4 /*yield*/, async_utils_1.callback2(db._query, {
query: "SELECT * FROM stats ORDER BY time DESC LIMIT 1",
})];
case 1:
res = _c.sent();
if (((_b = res === null || res === void 0 ? void 0 : res.rows) === null || _b === void 0 ? void 0 : _b.length) != 1) {
dbg("no data (1)");
return [2 /*return*/, null];
}
x = misc.map_without_undefined(res.rows[0]);
if (x == null) {
dbg("no data (2)");
return [2 /*return*/, null];
}
dbg("check_db_cache x = " + misc.to_json(x));
_stats_cached_db_query = new Date();
if (update && x.time < misc.seconds_ago(ttl - ttl_dt)) {
dbg("cache outdated -- will update stats");
return [2 /*return*/, null];
}
else {
dbg("using db stats from " + (new Date().getTime() - x.time) / 1000 + " secs ago.");
// storing still valid result in local cache
_stats_cached = misc.deep_copy(x);
return [2 /*return*/, _stats_cached];
}
return [3 /*break*/, 3];
case 2:
err_1 = _c.sent();
dbg("problem with query -- no stats in db?");
throw err_1;
case 3: return [2 /*return*/];
}
});
});
}
var running_projects_query = "SELECT count(*), run_quota -> 'member_host' AS member\nFROM projects\nWHERE state ->> 'state' in ('running', 'starting')\nGROUP BY member";
function calc_running_projects(db) {
return __awaiter(this, void 0, void 0, function () {
var data, res, _a, _b, row;
var e_1, _c;
return __generator(this, function (_d) {
switch (_d.label) {
case 0:
data = { free: 0, member: 0 };
return [4 /*yield*/, async_utils_1.callback2(db._query, { query: running_projects_query })];
case 1:
res = _d.sent();
try {
for (_a = __values(res.rows), _b = _a.next(); !_b.done; _b = _a.next()) {
row = _b.value;
if (row.member) {
data.member = parseInt(row.count);
}
else {
data.free = parseInt(row.count);
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
}
finally { if (e_1) throw e_1.error; }
}
return [2 /*return*/, data];
}
});
});
}
function _calc_stats(_a) {
var db = _a.db, dbg = _a.dbg, start_t = _a.start_t;
return __awaiter(this, void 0, void 0, function () {
var stats, R, K, _b, _c, _d, _e, _f, _g, _h, _j, tkey, _k, _l, _m, _o, _p, _q, _r, _s, e_2_1, _t, elapsed_t, duration_s;
var e_2, _u;
return __generator(this, function (_v) {
switch (_v.label) {
case 0:
stats = {
id: misc.uuid(),
time: new Date(),
accounts: 0,
projects: 0,
projects_created: {},
projects_edited: {},
accounts_created: {},
accounts_active: {},
files_opened: { distinct: {}, total: {} },
hub_servers: [],
running_projects: { free: 0, member: 0 },
};
R = schema_1.RECENT_TIMES;
K = schema_1.RECENT_TIMES_KEY;
_b = stats;
return [4 /*yield*/, _count_timespan(db, {
table: "accounts",
})];
case 1:
_b.accounts = _v.sent();
_c = stats;
return [4 /*yield*/, _count_timespan(db, {
table: "projects",
})];
case 2:
_c.projects = _v.sent();
_d = stats.projects_edited;
_e = K.active;
return [4 /*yield*/, _count_timespan(db, {
table: "projects",
field: "last_edited",
age_m: R.active,
})];
case 3:
_d[_e] = _v.sent();
_f = stats.accounts_active;
_g = K.active;
return [4 /*yield*/, _count_timespan(db, {
table: "accounts",
field: "last_active",
age_m: R.active,
})];
case 4:
_f[_g] = _v.sent();
return [4 /*yield*/, new Promise(function (done, reject) {
db._query({
query: "SELECT expire, host, clients FROM hub_servers",
cb: all_results(function (err, hub_servers) {
var e_3, _a;
if (err) {
reject(err);
}
else {
var now = new Date();
stats.hub_servers = [];
try {
for (var hub_servers_1 = __values(hub_servers), hub_servers_1_1 = hub_servers_1.next(); !hub_servers_1_1.done; hub_servers_1_1 = hub_servers_1.next()) {
var x = hub_servers_1_1.value;
if (x.expire > now) {
delete x.expire;
stats.hub_servers.push(x);
}
}
}
catch (e_3_1) { e_3 = { error: e_3_1 }; }
finally {
try {
if (hub_servers_1_1 && !hub_servers_1_1.done && (_a = hub_servers_1.return)) _a.call(hub_servers_1);
}
finally { if (e_3) throw e_3.error; }
}
done();
}
}),
});
})];
case 5:
_v.sent();
_v.label = 6;
case 6:
_v.trys.push([6, 16, 17, 18]);
_h = __values(["last_month", "last_week", "last_day", "last_hour"]), _j = _h.next();
_v.label = 7;
case 7:
if (!!_j.done) return [3 /*break*/, 15];
tkey = _j.value;
return [4 /*yield*/, _count_opened_files(db, {
age_m: R[tkey],
key: K[tkey],
data: stats.files_opened.distinct,
distinct: true,
})];
case 8:
_v.sent();
return [4 /*yield*/, _count_opened_files(db, {
age_m: R[tkey],
key: K[tkey],
data: stats.files_opened.total,
distinct: false,
})];
case 9:
_v.sent();
_k = stats.projects_edited;
_l = K[tkey];
return [4 /*yield*/, _count_timespan(db, {
table: "projects",
field: "last_edited",
age_m: R[tkey],
})];
case 10:
_k[_l] = _v.sent();
_m = stats.projects_created;
_o = K[tkey];
return [4 /*yield*/, _count_timespan(db, {
table: "projects",
field: "created",
age_m: R[tkey],
})];
case 11:
_m[_o] = _v.sent();
_p = stats.accounts_active;
_q = K[tkey];
return [4 /*yield*/, _count_timespan(db, {
table: "accounts",
field: "last_active",
age_m: R[tkey],
})];
case 12:
_p[_q] = _v.sent();
_r = stats.accounts_created;
_s = K[tkey];
return [4 /*yield*/, _count_timespan(db, {
table: "accounts",
field: "created",
age_m: R[tkey],
})];
case 13:
_r[_s] = _v.sent();
_v.label = 14;
case 14:
_j = _h.next();
return [3 /*break*/, 7];
case 15: return [3 /*break*/, 18];
case 16:
e_2_1 = _v.sent();
e_2 = { error: e_2_1 };
return [3 /*break*/, 18];
case 17:
try {
if (_j && !_j.done && (_u = _h.return)) _u.call(_h);
}
finally { if (e_2) throw e_2.error; }
return [7 /*endfinally*/];
case 18:
_t = stats;
return [4 /*yield*/, calc_running_projects(db)];
case 19:
_t.running_projects = _v.sent();
elapsed_t = process.hrtime(start_t);
duration_s = (elapsed_t[0] + elapsed_t[1] / 1e9).toFixed(4);
dbg("everything succeeded above after " + duration_s + " secs -- now insert stats");
// storing in local and db cache
_stats_cached = misc.deep_copy(stats);
return [4 /*yield*/, async_utils_1.callback2(db._query, {
query: "INSERT INTO stats",
values: stats,
})];
case 20:
_v.sent();
return [2 /*return*/, stats];
}
});
});
}
function calc_stats(db, opts) {
return __awaiter(this, void 0, void 0, function () {
var ttl_dt, ttl, ttl_db, update, cb, start_t, dbg, stats, err_2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
ttl_dt = opts.ttl_dt, ttl = opts.ttl, ttl_db = opts.ttl_db, update = opts.update, cb = opts.cb;
start_t = process.hrtime();
dbg = db._dbg("get_stats");
stats = null;
stats = check_local_cache({ update: update, ttl_dt: ttl_dt, ttl: ttl, ttl_db: ttl_db, dbg: dbg });
if (!(stats == null)) return [3 /*break*/, 2];
dbg("checking db cache?");
return [4 /*yield*/, check_db_cache({ db: db, update: update, ttl: ttl, ttl_dt: ttl_dt, dbg: dbg })];
case 1:
stats = _a.sent();
_a.label = 2;
case 2:
if (!(stats != null)) return [3 /*break*/, 3];
dbg("stats != null \u2192 nothing to do");
return [3 /*break*/, 8];
case 3:
if (!!update) return [3 /*break*/, 4];
dbg("warning: no recent stats but not allowed to update");
return [3 /*break*/, 8];
case 4:
dbg("we're actually recomputing the stats");
_a.label = 5;
case 5:
_a.trys.push([5, 7, , 8]);
return [4 /*yield*/, _calc_stats({ db: db, dbg: dbg, start_t: start_t })];
case 6:
stats = _a.sent();
return [3 /*break*/, 8];
case 7:
err_2 = _a.sent();
dbg("error calculating stats: err=" + err_2);
cb === null || cb === void 0 ? void 0 : cb(err_2, null);
return [2 /*return*/];
case 8:
dbg("stats=" + misc.to_json(stats) + ")");
// uncomment to fully debug the resulting stats object
//console.debug(JSON.stringify(stats, null, 2));
//process.exit();
cb === null || cb === void 0 ? void 0 : cb(undefined, stats);
return [2 /*return*/, stats];
}
});
});
}
exports.calc_stats = calc_stats;
//# sourceMappingURL=stats.js.map