@cocalc/project
Version:
CoCalc: project daemon
234 lines • 9.45 kB
JavaScript
"use strict";
/*
* This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
* License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.get_blob_store = exports.BlobStore = void 0;
const fs = __importStar(require("fs"));
const async_utils_node_1 = require("./async-utils-node");
const logger_1 = __importDefault(require("@cocalc/backend/logger"));
const misc_1 = require("@cocalc/util/misc");
const misc_node = require("@cocalc/backend/misc_node");
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
const express_1 = require("express");
const winston = (0, logger_1.default)("jupyter-blobs-sqlite");
const server_1 = require("@cocalc/project/project-status/server");
const JUPYTER_BLOBS_DB_FILE = process.env.JUPYTER_BLOBS_DB_FILE ??
`${process.env.SMC_LOCAL_HUB_HOME ?? process.env.HOME}/.jupyter-blobs-v0.db`;
// TODO: are these the only base64 encoded types that jupyter kernels return?
const BASE64_TYPES = ["image/png", "image/jpeg", "application/pdf", "base64"];
class BlobStore {
constructor() {
winston.debug("jupyter BlobStore: constructor");
try {
this.init();
winston.debug(`jupyter BlobStore: ${JUPYTER_BLOBS_DB_FILE} opened fine`);
}
catch (err) {
winston.debug(`jupyter BlobStore: ${JUPYTER_BLOBS_DB_FILE} open error - ${err}`);
// File may be corrupt/broken/etc. -- in this case, remove and try again.
// This database is only an image *cache*, so this is fine.
// See https://github.com/sagemathinc/cocalc/issues/2766
// Using sync is also fine, since this only happens once
// during initialization.
winston.debug("jupyter BlobStore: resetting database cache");
try {
fs.unlinkSync(JUPYTER_BLOBS_DB_FILE);
}
catch (error) {
err = error;
winston.debug(`Error trying to delete ${JUPYTER_BLOBS_DB_FILE}... ignoring: `, err);
}
this.init();
}
}
init() {
if (JUPYTER_BLOBS_DB_FILE == "memory") {
// as any, because @types/better-sqlite3 is not yet updated to support this
// doc about the constructor: https://wchargin.com/better-sqlite3/api.html#new-databasepath-options
this.db = new better_sqlite3_1.default(".db", { memory: true });
}
else {
this.db = new better_sqlite3_1.default(JUPYTER_BLOBS_DB_FILE);
}
this.init_table();
this.init_statements(); // table must exist!
if (JUPYTER_BLOBS_DB_FILE !== "memory") {
this.clean(); // do this once on start
this.db.exec("VACUUM");
}
}
init_table() {
this.db
.prepare("CREATE TABLE IF NOT EXISTS blobs (sha1 TEXT, data BLOB, type TEXT, ipynb TEXT, time INTEGER)")
.run();
}
init_statements() {
this.stmt_insert = this.db.prepare("INSERT INTO blobs VALUES(?, ?, ?, ?, ?)");
this.stmt_update = this.db.prepare("UPDATE blobs SET time=? WHERE sha1=?");
this.stmt_get = this.db.prepare("SELECT * FROM blobs WHERE sha1=?");
this.stmt_data = this.db.prepare("SELECT data FROM blobs where sha1=?");
this.stmt_keys = this.db.prepare("SELECT sha1 FROM blobs");
this.stmt_ipynb = this.db.prepare("SELECT ipynb, type, data FROM blobs where sha1=?");
}
clean() {
this.clean_old();
this.clean_filesize();
}
clean_old() {
// Delete anything old...
// The main point of this blob store being in the db is to ensure that when the
// project restarts, then user saves an ipynb,
// that they do not loose any work. So a few weeks should be way more than enough.
// Note that TimeTravel may rely on these old blobs, so images in TimeTravel may
// stop working after this long. That's a tradeoff.
this.db
.prepare("DELETE FROM blobs WHERE time <= ?")
.run((0, misc_1.months_ago)(1).getTime());
}
clean_filesize() {
// we also check for the actual filesize and in case, get rid of half of the old blobs
try {
const stats = fs.statSync(JUPYTER_BLOBS_DB_FILE);
const size_mb = stats.size / (1024 * 1024);
if (size_mb > 128) {
const cnt = this.db.prepare("SELECT COUNT(*) as cnt FROM blobs").get();
if (cnt?.cnt == null)
return;
const n = Math.floor(cnt.cnt / 2);
winston.debug(`jupyter BlobStore: large file of ${size_mb}MiB detected – deleting ${n} old rows.`);
if (n == 0)
return;
const when = this.db
.prepare("SELECT time FROM blobs ORDER BY time ASC LIMIT 1 OFFSET ?")
.get(n);
if (when?.time == null)
return;
winston.debug(`jupyter BlobStore: delete starting from ${when.time}`);
this.db.prepare("DELETE FROM blobs WHERE time <= ?").run(when.time);
}
}
catch (err) {
winston.debug(`jupyter BlobStore: clean_filesize error: ${err}`);
}
}
// used in testing
delete_all_blobs() {
this.db.prepare("DELETE FROM blobs").run();
}
// data could, e.g., be a uuencoded image
// We return the sha1 hash of it, and store it, along with a reference count.
// ipynb = (optional) text that is also stored and will be
// returned when get_ipynb is called
// This is used for some iframe support code.
save(data, type, ipynb) {
if (BASE64_TYPES.includes(type)) {
data = Buffer.from(data, "base64");
}
else {
data = Buffer.from(data);
}
const sha1 = misc_node.sha1(data);
const row = this.stmt_get.get(sha1);
if (row == null) {
this.stmt_insert.run([sha1, data, type, ipynb, Date.now()]);
}
else {
this.stmt_update.run([Date.now(), sha1]);
}
return sha1;
}
// Read a file from disk and save it in the database.
// Returns the sha1 hash of the file.
async readFile(path, type) {
return await this.save(await (0, async_utils_node_1.readFile)(path), type);
}
/*
free(sha1: string): void {
// instead, stuff gets freed 1 month after last save.
}
*/
// Return data with given sha1, or undefined if no such data.
get(sha1) {
const x = this.stmt_data.get(sha1);
if (x != null) {
return x.data;
}
}
get_ipynb(sha1) {
const row = this.stmt_ipynb.get(sha1);
if (row == null) {
return;
}
if (row.ipynb != null) {
return row.ipynb;
}
if (BASE64_TYPES.includes(row.type)) {
return row.data.toString("base64");
}
else {
return row.data.toString();
}
}
keys() {
return this.stmt_keys.all().map((x) => x.sha1);
}
express_router(base) {
const router = (0, express_1.Router)();
base += "blobs/";
router.get(base, (_, res) => {
res.send((0, misc_1.to_json)(this.keys()));
});
router.get(base + "*", (req, res) => {
const filename = req.path.slice(base.length);
const sha1 = `${req.query.sha1}`;
res.type(filename);
res.send(this.get(sha1));
});
return router;
}
}
exports.BlobStore = BlobStore;
let blob_store = undefined;
function get_blob_store() {
if (blob_store != null)
return blob_store;
try {
blob_store = new BlobStore();
(0, server_1.get_ProjectStatusServer)().clearComponentAlert("BlobStore");
return blob_store;
}
catch (err) {
(0, server_1.get_ProjectStatusServer)().setComponentAlert("BlobStore");
winston.warn(`unable to instantiate BlobStore -- ${err}`);
}
}
exports.get_blob_store = get_blob_store;
//# sourceMappingURL=jupyter-blobs-sqlite.js.map