helene
Version:
Real-time Web Apps for Node.js
221 lines • 8.75 kB
JavaScript
"use strict";
/**
* Handle every persistence-related task
* The interface Datastore expects to be implemented is
* * Persistence.loadDatabase(callback) and callback has signature err
* * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Persistence = void 0;
const indexes_1 = require("./indexes");
const serialization_1 = require("./serialization");
const custom_utils_1 = require("./custom-utils");
const collection_1 = require("./collection");
const throttle_1 = __importDefault(require("lodash/throttle"));
const lodash_1 = require("lodash");
const utils_1 = require("../utils");
class Persistence {
db;
inMemoryOnly;
name;
corruptAlertThreshold;
afterSerialization;
beforeDeserialization;
autocompactionIntervalId;
storage;
constructor(options) {
let i, j, randomString;
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.name = this.db.name;
this.corruptAlertThreshold =
options.corruptAlertThreshold !== undefined
? options.corruptAlertThreshold
: 0.1;
if (!this.inMemoryOnly &&
this.name &&
this.name.charAt(this.name.length - 1) === '~') {
throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
}
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss');
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss');
}
this.afterSerialization =
options.afterSerialization ||
function (s) {
return s;
};
this.beforeDeserialization =
options.beforeDeserialization ||
function (s) {
return s;
};
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = (0, custom_utils_1.uid)(i);
if (this.beforeDeserialization(this.afterSerialization(randomString)) !==
randomString) {
throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss');
}
}
}
}
async loadDatabase() {
this.db.resetIndexes();
if (this.inMemoryOnly) {
return null;
}
const rawData = await this.storage.read(this.name);
const treatedData = this.treatRawData(rawData);
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(key => {
this.db.indexes[key] = new indexes_1.Index(treatedData.indexes[key]);
});
// Fill cached database (i.e. all indexes) with data
try {
this.db.resetIndexes(treatedData.data);
}
catch (e) {
this.db.resetIndexes(); // Rollback any index which didn't fail
throw e;
}
await this.persistCachedDatabase();
}
throttleEmitUpdate = (0, throttle_1.default)(() => {
this.db.emit(collection_1.CollectionEvent.UPDATED);
}, 1000 / 20, { leading: true, trailing: true });
persistenceQueue = [];
/**
* This is the entry-point.
*/
async persistNewState(newDocs) {
this.throttleEmitUpdate();
const self = this;
let toPersist = '';
// In-memory only datastore
if (self.inMemoryOnly) {
return null;
}
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization((0, serialization_1.serialize)(doc)) + '\n';
});
if (toPersist.length === 0) {
return null;
}
await this.storage.append(self.name, toPersist);
}
queuedPersistState(newDocs) {
this.throttleEmitUpdate();
this.persistenceQueue.push(() => this.persistNewState(newDocs));
this.debouncedPersistState();
}
debouncedPersistState = (0, lodash_1.debounce)(async () => {
while (this.persistenceQueue.length > 0) {
await this.persistenceQueue.shift()();
await (0, utils_1.sleep)(1);
}
}, utils_1.Environment.isTest ? 1 : 1000);
async persistCachedDatabase() {
const self = this;
let toPersist = '';
if (this.inMemoryOnly) {
return null;
}
this.db.getAllData().forEach(doc => {
toPersist += this.afterSerialization((0, serialization_1.serialize)(doc)) + '\n';
});
Object.keys(this.db.indexes).forEach(fieldName => {
if (fieldName != '_id') {
// The special _id index is managed by Collection, the others need to be persisted
toPersist +=
self.afterSerialization((0, serialization_1.serialize)({
$$indexCreated: {
fieldName: fieldName,
unique: this.db.indexes[fieldName].unique,
sparse: this.db.indexes[fieldName].sparse,
},
})) + '\n';
}
});
await this.storage.write(this.name, toPersist);
this.db.emit(collection_1.CollectionEvent.COMPACTED);
}
/**
* Queue a rewrite of the datafile
*/
async compactDatafile() {
return this.persistCachedDatabase();
}
/**
* Set automatic compaction every interval ms
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
setAutocompactionInterval(interval) {
const minInterval = 5000;
const realInterval = Math.max(interval || 0, minInterval);
this.stopAutocompaction();
this.autocompactionIntervalId = setInterval(() => {
this.compactDatafile().catch(console.error);
}, realInterval);
}
/**
* Stop autocompaction (do nothing if autocompaction was not running)
*/
stopAutocompaction() {
if (this.autocompactionIntervalId) {
clearInterval(this.autocompactionIntervalId);
}
}
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
treatRawData(rawData) {
const data = rawData?.split('\n') || [], dataById = {}, tdata = [], indexes = {};
let corruptItems = -1; // Last line of every data file is usually blank so not really corrupt
for (let i = 0; i < data.length; i += 1) {
let doc;
try {
doc = (0, serialization_1.deserialize)(this.beforeDeserialization(data[i]));
if (doc._id !== undefined) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
}
else {
dataById[doc._id] = doc;
}
}
else if (doc.$$indexCreated &&
doc.$$indexCreated.fieldName != undefined) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated;
}
else if (typeof doc.$$indexRemoved === 'string') {
delete indexes[doc.$$indexRemoved];
}
}
catch (e) {
corruptItems += 1;
}
}
// A bit lenient on corruption
if (data.length > 0 &&
corruptItems / data.length > this.corruptAlertThreshold) {
throw new Error('More than ' +
Math.floor(100 * this.corruptAlertThreshold) +
'% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss');
}
Object.keys(dataById).forEach(function (k) {
tdata.push(dataById[k]);
});
return { data: tdata, indexes };
}
}
exports.Persistence = Persistence;
//# sourceMappingURL=persistence.js.map