@iobroker/db-objects-redis
Version:
The Library contains the Database classes for Redis based objects database client.
1,085 lines • 173 kB
JavaScript
/**
* Object DB in REDIS - Client
*
* MIT License
* Written by bluefox <dogafox@gmail.com>, 2014-2024
*
*/
// @ts-expect-error no ts module
import extend from 'node.extend';
import Redis from 'ioredis';
import { tools } from '@iobroker/db-base';
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { isDeepStrictEqual } from 'node:util';
import deepClone from 'deep-clone';
import * as utils from '../../lib/objects/objectsUtils.js';
import semver from 'semver';
import * as CONSTS from '../../lib/objects/constants.js';
import * as url from 'node:url';
// eslint-disable-next-line unicorn/prefer-module
const thisDir = url.fileURLToPath(new URL('.', import.meta.url || `file://${__filename}`));
const ERRORS = CONSTS.ERRORS;
export class ObjectsInRedisClient {
client;
fileNamespace;
redisNamespace;
fileNamespaceL;
objNamespace;
setNamespace;
metaNamespace;
objNamespaceL;
supportedProtocolVersions;
stop;
sub;
subSystem;
settings;
preserveSettings;
defaultNewAcl;
namespace;
hostname;
scripts;
existingMetaObjects;
log;
activeProtocolVersion;
useSets;
noLegacyMultihost;
userSubscriptions;
systemSubscriptions;
constructor(settings) {
this.settings = settings || {};
this.redisNamespace = `${this.settings.redisNamespace || this.settings.connection?.redisNamespace || 'cfg'}.`;
this.fileNamespace = `${this.redisNamespace}f.`;
this.fileNamespaceL = this.fileNamespace.length;
this.objNamespace = `${this.redisNamespace}o.`;
this.setNamespace = `${this.redisNamespace}s.`;
this.metaNamespace = `${this.settings.metaNamespace || 'meta'}.`;
this.objNamespaceL = this.objNamespace.length;
this.supportedProtocolVersions = ['4'];
this.stop = false;
this.client = null;
this.sub = null;
this.subSystem = null;
this.preserveSettings = ['custom', 'smartName', 'material', 'habpanel', 'mobile'];
this.defaultNewAcl = this.settings.defaultNewAcl || null;
this.namespace = this.settings.namespace || this.settings.hostname || '';
this.hostname = this.settings.hostname || tools.getHostName();
this.scripts = {};
this.userSubscriptions = {};
this.systemSubscriptions = {};
// cached meta-objects for file operations
this.existingMetaObjects = {};
this.log = tools.getLogger(this.settings.logger);
if (this.settings.autoConnect !== false) {
this.connectDb();
}
}
/**
* Checks if we are allowed to start and sets the protocol version accordingly
*/
async _determineProtocolVersion() {
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
let protoVersion;
try {
protoVersion = await this.client.get(`${this.metaNamespace}objects.protocolVersion`);
}
catch (e) {
if (e.message.includes('GET-UNSUPPORTED')) {
// secondary updated and primary < 4.0
return;
}
}
if (!protoVersion) {
// if no proto version existent yet, we set ours
const highestVersion = Math.max(...this.supportedProtocolVersions.map(value => parseInt(value)));
await this.setProtocolVersion(highestVersion);
this.activeProtocolVersion = highestVersion.toString();
return;
}
// check if we can support this version
if (this.supportedProtocolVersions.includes(protoVersion)) {
this.activeProtocolVersion = protoVersion;
}
else {
throw new Error(`This host does not support protocol version "${protoVersion}"`);
}
}
connectDb() {
this.settings.connection = this.settings.connection || {};
const onChange = this.settings.change; // on change handler
const onChangeUser = this.settings.changeUser; // on change handler for User events
const onChangeFileUser = this.settings.changeFileUser; // on change handler for User file events
this.settings.connection.options = this.settings.connection.options || {};
const retry_max_delay = this.settings.connection.options.retry_max_delay || 5_000;
const retry_max_count = this.settings.connection.options.retry_max_count || 19;
let ready = false;
let initError = false;
let connected = false;
let reconnectCounter = 0;
let errorLogged = false;
this.settings.connection.options.retryStrategy = (reconnectCount) => {
if (!ready && initError) {
return new Error('No more tries');
}
if (this.stop) {
return new Error('Client has stopped ... no retries anymore');
}
if (ready && reconnectCount >= retry_max_count) {
return new Error('Stop trying to reconnect');
}
// A function that receives an options object as parameter including the retry attempt,
// the total_retry_time indicating how much time passed since the last time connected,
// the error why the connection was lost and the number of times_connected in total.
// If you return a number from this function, the retry will happen exactly after that
// time in milliseconds. If you return a non-number, no further retry will happen and
// all offline commands are flushed with errors. Return an error to return that
// specific error to all offline commands.
if (!ready) {
return 300;
}
return retry_max_delay;
};
delete this.settings.connection.options.retry_max_delay;
this.settings.connection.options.enableReadyCheck = true;
if (this.settings.connection.port === 0) {
// Port = 0 means unix socket
// initiate a unix socket connection
this.settings.connection.options.path = this.settings.connection.host;
this.log.debug(`${this.namespace} Redis Objects: Use File Socket for connection: ${this.settings.connection.options.path}`);
}
else if (Array.isArray(this.settings.connection.host)) {
const configuredPort = this.settings.connection.port;
// Host is an array means we use a sentinel
const defaultPort = Array.isArray(configuredPort) ? null : configuredPort;
this.settings.connection.options.sentinels = this.settings.connection.host.map((redisNode, idx) => ({
host: redisNode,
// @ts-expect-error ts does not get that if defPort is null we have an array
port: defaultPort === null ? configuredPort[idx] : defaultPort,
}));
this.settings.connection.options.name = this.settings.connection.sentinelName
? this.settings.connection.sentinelName
: 'mymaster';
this.log.debug(`${this.namespace} Redis Objects: Use Sentinel for connection: ${this.settings.connection.options.name}, ${JSON.stringify(this.settings.connection.options.sentinels)}`);
}
else {
this.settings.connection.options.host = this.settings.connection.host;
this.settings.connection.options.port = this.settings.connection.port;
this.log.debug(`${this.namespace} Redis Objects: Use Redis connection: ${this.settings.connection.options.host}:${this.settings.connection.options.port}`);
}
this.settings.connection.options.db = this.settings.connection.options.db || 0;
this.settings.connection.options.family = this.settings.connection.options.family || 0;
this.settings.connection.options.password =
this.settings.connection.options.auth_pass || this.settings.connection.pass || null;
this.settings.connection.options.autoResubscribe = false; // We do our own resubscribe because other sometimes not work
// REDIS does not allow whitespaces, we have some because of pid
this.settings.connection.options.connectionName = this.namespace.replace(/\s/g, '');
this.client = new Redis(this.settings.connection.options);
this.client.on('error', error => {
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Redis ERROR Objects: (${this.stop}) ${error.message} / ${error.stack}`);
}
if (this.stop) {
return;
}
if (!ready) {
initError = true;
// It seems we have a socket.io server
if (error.message.startsWith('Protocol error, got "H" as reply type byte.')) {
this.log.error(`${this.namespace} Could not connect to objects database at ${this.settings.connection.options.host}:${this.settings.connection.options.port} (invalid protocol). Please make sure the configured IP and port points to a host running JS-Controller >= 2.0. and that the port is not occupied by other software!`);
}
return;
}
this.log.error(`${this.namespace} Objects database error: ${error.message}`);
errorLogged = true;
});
this.client.on('end', () => {
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Objects-Redis Event end (stop=${this.stop})`);
}
if (ready && typeof this.settings.disconnected === 'function') {
this.settings.disconnected();
}
});
this.client.on('connect', () => {
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Objects-Redis Event connect (stop=${this.stop})`);
}
connected = true;
if (errorLogged) {
this.log.info(`${this.namespace} Objects database successfully reconnected`);
errorLogged = false;
}
});
this.client.on('close', () => {
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Objects-Redis Event close (stop=${this.stop})`);
}
//if (ready && typeof this.settings.disconnected === 'function') this.settings.disconnected();
});
this.client.on('reconnecting', () => {
if (connected && !ready && !initError) {
reconnectCounter++;
}
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Objects-Redis Event reconnect (reconnectCounter=${reconnectCounter}, stop=${this.stop})`);
}
if (reconnectCounter > 2) {
// fallback logic for nodejs <10
this.log.error(`${this.namespace} The DB port ${this.settings.connection.options.port} is occupied by something that is not a Redis protocol server. Please check other software running on this port or, if you use iobroker, make sure to update to js-controller 2.0 or higher!`);
return;
}
connected = false;
initError = false;
});
this.client.on('ready', async () => {
if (this.stop || !this.client) {
return;
}
initError = false;
this.log.debug(`${this.namespace} Objects client ready ... initialize now`);
try {
await this.client.config('SET', 'lua-time-limit', 10000); // increase LUA timeout TODO still necessary with scan?
}
catch (e) {
this.log.warn(`${this.namespace} Unable to increase LUA script timeout: ${e.message}`);
}
let initCounter = 0;
if (!this.subSystem && typeof onChange === 'function') {
initCounter++;
this.log.debug(`${this.namespace} Objects create System PubSub Client`);
this.subSystem = new Redis(this.settings.connection.options);
if (typeof this.settings.primaryHostLost === 'function') {
try {
// enable Expiry/Evicted events in server - same as states (could be same db)
await this.client.config('SET', 'notify-keyspace-events', 'Exe');
}
catch (e) {
this.log.warn(`${this.namespace} Unable to enable Expiry Keyspace events from Redis Server: ${e.message}`);
}
this.subSystem.on('message', (channel, message) => {
if (channel === `__keyevent@${this.settings.connection.options.db}__:expired` ||
channel === `__keyevent@${this.settings.connection.options.db}__:evicted`) {
this.log.silly(`${this.namespace} redis message expired/evicted ${channel}:${message}`);
if (message === `${this.metaNamespace}objects.primaryHost` &&
typeof this.settings.primaryHostLost === 'function') {
this.settings.primaryHostLost();
}
}
});
}
if (typeof onChange === 'function') {
this.subSystem.on('pmessage', (pattern, channel, message) => setImmediate(() => {
this.log.silly(`${this.namespace} Objects system redis pmessage ${pattern}/${channel}:${message}`);
if (channel.startsWith(this.metaNamespace)) {
if (channel === `${this.metaNamespace}objects.protocolVersion` &&
message !== this.activeProtocolVersion) {
if (typeof this.settings.disconnected === 'function') {
// protocol version has changed, restart controller
this.log.info(`${this.namespace} Objects protocol version has changed, disconnecting!`);
this.settings.disconnected();
}
}
else if (channel === `${this.metaNamespace}objects.features.useSets`) {
const newUseSets = !!parseInt(message);
if (newUseSets !== this.useSets) {
this.log.info(`${this.namespace} Sets ${newUseSets ? 'activated' : 'deactivated'}: restarting ...`);
this.useSets = newUseSets;
// luas are no longer up to date, lets restart
if (typeof this.settings.disconnected === 'function') {
this.settings.disconnected();
}
}
}
return;
}
try {
if (channel.startsWith(this.objNamespace) && channel.length > this.objNamespaceL) {
const id = channel.substring(this.objNamespaceL);
try {
const obj = message ? JSON.parse(message) : null;
if (id === 'system.config' &&
obj?.common?.defaultNewAcl &&
!isDeepStrictEqual(obj.common.defaultNewAcl, this.defaultNewAcl)) {
this.defaultNewAcl = deepClone(obj.common.defaultNewAcl);
if (this.settings.controller) {
this.setDefaultAcl(this.defaultNewAcl);
}
}
onChange(id, obj);
}
catch (e) {
this.log.warn(`${this.namespace} Objects Cannot process system pmessage ${id} - ${message}: ${e.message}`);
this.log.warn(`${this.namespace} ${e.stack}`);
}
}
else {
this.log.warn(`${this.namespace} Objects Received unexpected system pmessage: ${channel}`);
}
}
catch (e) {
this.log.warn(`${this.namespace} Objects system pmessage ${channel} ${JSON.stringify(message)} ${e.message}`);
this.log.warn(`${this.namespace} ${e.stack}`);
}
}));
}
this.subSystem.on('end', () => {
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Objects-Redis System Event end sub (stop=${this.stop})`);
}
if (ready && typeof this.settings.disconnected === 'function') {
this.settings.disconnected();
}
});
this.subSystem.on('error', error => {
if (this.stop) {
return;
}
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} PubSub System client Objects No redis connection: ${JSON.stringify(error)}`);
}
});
if (this.settings.connection.enhancedLogging) {
this.subSystem.on('connect', () => this.log.silly(`${this.namespace} PubSub System client Objects-Redis Event connect (stop=${this.stop})`));
this.subSystem.on('close', () => this.log.silly(`${this.namespace} PubSub System client Objects-Redis Event close (stop=${this.stop})`));
this.subSystem.on('reconnecting', reconnectCounter => this.log.silly(`${this.namespace} PubSub System client Objects-Redis Event reconnect (reconnectCounter=${reconnectCounter}, stop=${this.stop})`));
}
this.subSystem.on('ready', async () => {
if (--initCounter < 1) {
if (this.settings.connection.port === 0) {
this.log.debug(`${this.namespace} Objects ${ready ? 'system re' : ''}connected to redis: ${tools.maybeArrayToString(this.settings.connection.host)}`);
}
else {
this.log.debug(`${this.namespace} Objects ${ready ? 'system re' : ''}connected to redis: ${tools.maybeArrayToString(this.settings.connection.host)}:${tools.maybeArrayToString(this.settings.connection.port)}`);
}
if (!ready && typeof this.settings.connected === 'function') {
this.settings.connected();
}
ready = true;
}
// subscribe on system.config anytime because also adapters need stuff like defaultNewAcl (especially admin)
try {
if (this.subSystem) {
await this.subSystem.psubscribe(`${this.objNamespace}system.config`);
}
}
catch {
// ignore
}
// subscribe to meta changes
try {
if (this.subSystem) {
await this.subSystem.psubscribe(`${this.metaNamespace}*`);
}
}
catch (e) {
this.log.warn(`${this.namespace} Unable to subscribe to meta namespace "${this.metaNamespace}" changes: ${e.message}`);
}
if (this.subSystem) {
for (const sub of Object.keys(this.systemSubscriptions)) {
try {
await this.subSystem.psubscribe(sub);
}
catch {
// ignore
}
}
}
});
}
if (!this.sub && (typeof onChangeUser === 'function' || typeof onChangeFileUser === 'function')) {
initCounter++;
this.log.debug(`${this.namespace} Objects create User PubSub Client`);
this.sub = new Redis(this.settings.connection.options);
this.sub.on('pmessage', (pattern, channel, message) => {
setImmediate(() => {
this.log.silly(`${this.namespace} Objects user redis pmessage ${pattern}/${channel}:${message}`);
try {
if (channel.startsWith(this.objNamespace) && channel.length > this.objNamespaceL) {
if (onChangeUser) {
const id = channel.substring(this.objNamespaceL);
try {
const obj = message ? JSON.parse(message) : null;
onChangeUser(id, obj);
}
catch (e) {
this.log.warn(`${this.namespace} Objects user cannot process pmessage ${id} - ${message}: ${e.message}`);
this.log.warn(`${this.namespace} ${e.stack}`);
}
}
}
else if (channel.startsWith(this.fileNamespace) && channel.length > this.fileNamespaceL) {
if (onChangeFileUser) {
// cfg.f.vis-2.0$%$main/historyChart.js$%$data
const [id, fileName] = channel.substring(this.fileNamespaceL).split('$%$');
try {
const obj = message ? JSON.parse(message) : null;
onChangeFileUser(id, fileName, obj);
}
catch (e) {
this.log.warn(`${this.namespace} Objects user cannot process pmessage ${id}/${fileName} - ${message}: ${e.message}`);
this.log.warn(`${this.namespace} ${e.stack}`);
}
}
}
else {
this.log.warn(`${this.namespace} Objects user received unexpected pmessage: ${channel}`);
}
}
catch (e) {
this.log.warn(`${this.namespace} Objects user pmessage ${channel} ${JSON.stringify(message)} ${e.message}`);
this.log.warn(`${this.namespace} ${e.stack}`);
}
});
});
this.sub.on('end', () => {
if (this.settings.connection.enhancedLogging) {
this.log.silly(`${this.namespace} Objects-Redis Event end user sub (stop=${this.stop})`);
}
if (ready && typeof this.settings.disconnected === 'function') {
this.settings.disconnected();
}
});
this.sub.on('error', error => {
if (this.stop) {
return;
}
this.settings.connection.enhancedLogging &&
this.log.silly(`${this.namespace} PubSub user client Objects No redis connection: ${JSON.stringify(error)}`);
});
if (this.settings.connection.enhancedLogging) {
this.sub.on('connect', () => this.log.silly(`${this.namespace} PubSub user client Objects-Redis Event connect (stop=${this.stop})`));
this.sub.on('close', () => this.log.silly(`${this.namespace} PubSub user client Objects-Redis Event close (stop=${this.stop})`));
this.sub.on('reconnecting', reconnectCounter => this.log.silly(`${this.namespace} PubSub user client Objects-Redis Event reconnect (reconnectCounter=${reconnectCounter}, stop=${this.stop})`));
}
this.sub.on('ready', async () => {
if (!this.sub) {
// client gone while ready emitted, can maybe not happen but ts is happy
return;
}
if (--initCounter < 1) {
if (this.settings.connection.port === 0) {
this.log.debug(`${this.namespace} Objects ${ready ? 'user re' : ''}connected to redis: ${tools.maybeArrayToString(this.settings.connection.host)}`);
}
else {
this.log.debug(`${this.namespace} Objects ${ready ? 'user re' : ''}connected to redis: ${tools.maybeArrayToString(this.settings.connection.host)}:${tools.maybeArrayToString(this.settings.connection.port)}`);
}
!ready && typeof this.settings.connected === 'function' && this.settings.connected();
ready = true;
}
for (const sub of Object.keys(this.userSubscriptions)) {
try {
await this.sub.psubscribe(sub);
}
catch {
// ignore
}
}
});
}
if (!this.client) {
return;
}
// do this before starting with async calls ;-)
initCounter++;
try {
// check if we are allowed to use sets
this.useSets = !!parseInt((await this.client.get(`${this.metaNamespace}objects.features.useSets`)) || '0');
}
catch (e) {
// if unsupported we have a legacy host
if (!e.message.includes('UNSUPPORTED')) {
this.log.error(`${this.namespace} Cannot determine Set feature status: ${e.message}`);
return;
}
this.useSets = false;
}
try {
await this._determineProtocolVersion();
}
catch (e) {
this.log.error(`${this.namespace} ${e.message}`);
throw new Error('Objects DB is not allowed to start in the current Multihost environment');
}
// for controller v4 we have to check if we can use the new lua scripts and set logic
// TODO: remove this backward shim if controller v4.0 is old enough
let keys = await this._getKeysViaScan(`${this.objNamespace}system.host.*`);
// filter out obvious non-host objects
const hostRegex = new RegExp(`^${this.objNamespace.replace(/\./g, '\\.')}system\\.host\\.[^.]+$`);
keys = keys.filter(id => hostRegex.test(id));
/** if false we have a host smaller 4 (no proto version for this existing) */
this.noLegacyMultihost = true;
try {
if (keys.length) {
// else no host known yet - so we are single host
const objs = await this.client.mget(keys);
for (const strObj of objs) {
const obj = strObj !== null ? JSON.parse(strObj) : strObj;
if (obj &&
obj.type === 'host' &&
obj._id !== `system.host.${this.hostname}` &&
obj.common &&
obj.common.installedVersion &&
semver.lt(obj.common.installedVersion, '4.0.0')) {
// one of the host has a version smaller 4, we have to use legacy db
this.noLegacyMultihost = false;
this.log.info(`${this.namespace} Sets unsupported`);
}
}
}
}
catch (e) {
this.log.error(`${this.namespace} Cannot determine Lua scripts strategy: ${e.message} ${JSON.stringify(keys)}`);
return;
}
this.log.debug(`${this.namespace} Objects client initialize lua scripts`);
try {
await this.loadLuaScripts();
}
catch (err) {
this.log.error(`${this.namespace} Cannot initialize database scripts: ${err.message}`);
return;
}
// init default new acl
let obj;
try {
obj = await this.client.get(`${this.objNamespace}system.config`);
}
catch {
// ignore
}
if (obj) {
try {
obj = JSON.parse(obj);
}
catch {
this.log.error(`${this.namespace} Cannot parse JSON system.config: ${obj}`);
obj = null;
}
if (obj && obj.common && obj.common.defaultNewAcl) {
this.defaultNewAcl = obj.common.defaultNewAcl;
}
}
else {
this.log.error(`${this.namespace} Cannot read system.config: ${obj} (OK when migrating or restoring)`);
}
if (--initCounter < 1) {
if (this.settings.connection.port === 0) {
this.log.debug(`${this.namespace} Objects ${ready ? 'client re' : ''}connected to redis: ${tools.maybeArrayToString(this.settings.connection.host)}`);
}
else {
this.log.debug(`${this.namespace} Objects ${ready ? 'client re' : ''}connected to redis: ${tools.maybeArrayToString(this.settings.connection.host)}:${tools.maybeArrayToString(this.settings.connection.port)}`);
}
!ready && typeof this.settings.connected === 'function' && this.settings.connected();
ready = true;
}
});
}
getStatus() {
return { type: 'redis', server: false };
}
/**
* Checks if given ID is a meta-object, else throws error
*
* @param id to check
* @throws Error if id is invalid
*/
async validateMetaObject(id) {
if (this.existingMetaObjects[id] === undefined) {
// if not cached -> getObject
const obj = await this.getObject(id);
if (obj && obj.type === 'meta') {
this.existingMetaObjects[id] = true;
}
else {
this.existingMetaObjects[id] = false;
throw new Error(`${id} is not an object of type "meta"`);
}
}
else if (this.existingMetaObjects[id] === false) {
throw new Error(`${id} is not an object of type "meta"`);
}
}
normalizeFilename(name) {
return name ? name.replace(/[/\\]+/g, '/') : name;
}
// -------------- FILE FUNCTIONS -------------------------------------------
/**
* Sets a buffer to the Redis DB
*
* @param id id of the file
* @param data content, if string is passed it will be converted to a Buffer
*/
async _setBinaryState(id, data) {
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
if (!Buffer.isBuffer(data)) {
data = Buffer.from(data);
}
await this.client.set(id, data);
await this.client.publish(id, data.byteLength.toString(10));
}
/**
* get buffer of given id from redis
*
* @param id - id of the data with namespace prefix
*/
_getBinaryState(id) {
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
return this.client.getBuffer(id);
}
/**
* deletes binary state of given id from redis db
*
* @param id - id to delete, with namespace prefix
*/
async _delBinaryState(id) {
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
else {
await this.client.del(id);
await this.client.publish(id, 'null'); // inform about deletion
}
}
getFileId(id, name, isMeta) {
name = this.normalizeFilename(name);
// e.g. ekey.admin and admin/ekey.png
if (id.endsWith('.admin')) {
if (name.startsWith('admin/')) {
name = name.replace(/^admin\//, '');
}
else if (name.match(/^iobroker.[-\d\w]\/admin\//i)) {
// e.g. ekey.admin and iobroker.ekey/admin/ekey.png
name = name.replace(/^iobroker.[-\d\w]\/admin\//i, '');
}
}
let normalized;
try {
normalized = utils.sanitizePath(id, name);
}
catch {
this.log.debug(`${this.namespace} Invalid file path ${id}/${name}`);
return '';
}
if (id !== '*') {
id = normalized.id;
}
name = normalized.name;
return `${this.fileNamespace + id}$%$${name}${isMeta !== undefined ? (isMeta ? '$%$meta' : '$%$data') : ''}`;
}
async checkFile(id, name, options, flag, callback) {
// read file settings from redis
const fileId = this.getFileId(id, name, true);
if (!fileId) {
const fileOptions = { notExists: true };
if (utils.checkFile(fileOptions, options, flag, this.defaultNewAcl)) {
return tools.maybeCallback(callback, false, options, fileOptions); // NO error
}
return tools.maybeCallback(callback, true, options); // error
}
if (!this.client) {
// @ts-expect-error TODO: not in specs, better just maybe cb check false?
return tools.maybeCallbackWithRedisError(callback, ERRORS.ERROR_DB_CLOSED, options);
}
let fileOptions;
try {
fileOptions = await this.client.get(fileId);
}
catch {
// ignore
}
fileOptions = fileOptions || '{"notExists": true}';
try {
fileOptions = JSON.parse(fileOptions);
}
catch {
this.log.error(`${this.namespace} Cannot parse JSON ${id}: ${fileOptions}`);
fileOptions = { notExists: true };
}
if (utils.checkFile(fileOptions, options, flag, this.defaultNewAcl)) {
return tools.maybeCallback(callback, false, options, fileOptions); // NO error
}
return tools.maybeCallback(callback, true, options); // error
}
checkFileRights(id, name, options, flag, callback) {
return utils.checkFileRights(this, id, name, options, flag, callback);
}
async _setDefaultAcl(ids, defaultAcl) {
for (const id of ids) {
try {
const obj = await this.getObject(id);
if (obj && !obj.acl) {
obj.acl = defaultAcl;
await this.setObject(id, obj, null);
}
}
catch (e) {
this.log.error(`${this.namespace} _setDefaultAcl error on id "${id}" with acl "${JSON.stringify(defaultAcl)}": ${e.message}`);
}
}
}
async setDefaultAcl(defaultNewAcl) {
this.defaultNewAcl = defaultNewAcl || {
owner: CONSTS.SYSTEM_ADMIN_USER,
ownerGroup: CONSTS.SYSTEM_ADMIN_GROUP,
object: 0x664,
state: 0x664,
file: 0x664,
};
try {
// Get ALL Objects
const ids = await this.getKeysAsync('*');
if (ids) {
await this._setDefaultAcl(ids, this.defaultNewAcl);
}
}
catch (e) {
this.log.error(`${this.namespace} Could not update default acl: ${e.message}`);
}
}
getUserGroup(user, callback) {
return utils.getUserGroup(this, user, (error, user, userGroups, userAcl) => {
if (error) {
this.log.error(`${this.namespace} ${error.stack}`);
}
return tools.maybeCallback(callback, user, userGroups, userAcl);
});
}
async _writeFile(id, name, data, options, callback, meta) {
const matchedExtension = name.match(/\.[^.]+$/);
const ext = matchedExtension ? matchedExtension[0] : '';
const isTextData = typeof data === 'string';
const { mimeType, isBinary } = utils.getMimeType(ext, isTextData);
const metaID = this.getFileId(id, name, true);
if (!this.client) {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_DB_CLOSED);
}
// virtual files only get Meta objects
if (options.virtualFile) {
meta = {
notExists: true,
virtualFile: true,
}; // Store file with flags as it would not exist
try {
await this.client.set(metaID, JSON.stringify(meta));
return tools.maybeCallback(callback);
}
catch (e) {
return tools.maybeCallbackWithRedisError(callback, e);
}
}
else {
if (!meta) {
meta = { createdAt: Date.now() };
}
if (!meta.acl) {
meta.acl = {
owner: options.user || (this.defaultNewAcl && this.defaultNewAcl.owner) || CONSTS.SYSTEM_ADMIN_USER,
ownerGroup: options.group ||
(this.defaultNewAcl && this.defaultNewAcl.ownerGroup) ||
CONSTS.SYSTEM_ADMIN_GROUP,
permissions: options.mode || (this.defaultNewAcl && this.defaultNewAcl.file) || 0x644,
};
}
meta.stats = {
size: data ? data.length : 0,
};
if (Object.prototype.hasOwnProperty.call(meta, 'notExists')) {
delete meta.notExists;
}
meta.mimeType = options.mimeType || mimeType;
meta.binary = isBinary;
meta.acl.ownerGroup =
meta.acl.ownerGroup ||
(this.defaultNewAcl && this.defaultNewAcl.ownerGroup) ||
CONSTS.SYSTEM_ADMIN_GROUP;
meta.modifiedAt = Date.now();
try {
await this._setBinaryState(this.getFileId(id, name, false), data);
await this.client.set(metaID, JSON.stringify(meta));
return tools.maybeCallback(callback);
}
catch (e) {
return tools.maybeCallbackWithRedisError(callback, e);
}
}
}
async writeFile(id, name, data, options, callback) {
if (typeof options === 'function') {
callback = options;
options = null;
}
if (typeof options === 'string') {
options = { mimeType: options };
}
if (options?.acl) {
options.acl = null;
}
if (!callback) {
return this.writeFileAsync(id, name, data, options);
}
try {
await this.validateMetaObject(id);
}
catch (e) {
this.log.error(`${this.namespace} Cannot write file ${name}: ${e.message}`);
return tools.maybeCallbackWithError(callback, e);
}
if (typeof name !== 'string' || !name.length || name === '/') {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_NOT_FOUND);
}
if (name.startsWith('/')) {
name = name.substring(1);
}
if (data === undefined) {
data = null;
}
// If file yet exists => check the permissions
return this.checkFileRights(id, name, options, CONSTS.ACCESS_WRITE, (err, options, meta) => {
if (err) {
return tools.maybeCallbackWithError(callback, err);
}
return this._writeFile(id, name, data, options, callback, meta);
});
}
writeFileAsync(id, name, data, options) {
return new Promise((resolve, reject) => this.writeFile(id, name, data, options, err => (err ? reject(err) : resolve())));
}
async _readFile(id, name, meta) {
if (meta.notExists) {
throw new Error(ERRORS.ERROR_NOT_FOUND);
}
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
let buffer;
buffer = await this._getBinaryState(this.getFileId(id, name, false));
const mimeType = meta?.mimeType;
if (meta && !meta.binary && buffer) {
buffer = buffer.toString();
}
return { file: buffer, mimeType: mimeType };
}
readFile(id, name, options, callback) {
if (typeof options === 'function') {
callback = options;
options = null;
}
if (options?.acl) {
options.acl = null;
}
if (!callback) {
return new Promise((resolve, reject) => this.readFile(id, name, options, (err, res, mimeType) => err ? reject(err) : resolve({ file: res, mimeType: mimeType })));
}
if (typeof name !== 'string' || !name.length || name === '/') {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_NOT_FOUND);
}
if (name.startsWith('/')) {
name = name.substring(1);
}
options = options || {};
this.checkFileRights(id, name, options, CONSTS.ACCESS_READ, async (err, options, meta) => {
if (err) {
return tools.maybeCallbackWithError(callback, err);
}
try {
const { file, mimeType } = await this._readFile(id, name, meta);
return tools.maybeCallbackWithError(callback, null, file, mimeType);
}
catch (e) {
return tools.maybeCallbackWithError(callback, e);
}
});
}
/**
* Check if given object exists
*
* @param id id of the object
* @param options optional user context
*/
async objectExists(id, options) {
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
if (!id || typeof id !== 'string') {
throw new Error(`invalid id ${JSON.stringify(id)}`);
}
try {
await new Promise((resolve, reject) => {
utils.checkObjectRights(this, null, null, options, CONSTS.ACCESS_LIST, err => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
const exists = await this.client.exists(this.objNamespace + id);
return !!exists;
}
catch (e) {
this.log.error(`${this.namespace} Cannot check object existence of "${id}": ${e.message}`);
return Promise.reject(new Error(`Cannot check object existence of "${id}": ${e.message}`));
}
}
/**
* Check if given file exists
*
* @param id id of the namespace
* @param name name of the file
* @param options optional user context
*/
async fileExists(id, name, options) {
if (typeof name !== 'string') {
name = '';
}
if (name.startsWith('/')) {
name = name.substring(1);
}
try {
await new Promise((resolve, reject) => {
this.checkFileRights(id, name, options, CONSTS.ACCESS_READ, err => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
id = this.getFileId(id, name, false);
const exists = await this.client.exists(id);
return !!exists;
}
catch (e) {
this.log.error(`${this.namespace} Cannot check file existence of "${id}": ${e.message}`);
throw new Error(`Cannot check file existence of "${id}": ${e.message}`);
}
}
async _unlink(id, name, options, meta) {
if (!this.client) {
throw new Error(ERRORS.ERROR_DB_CLOSED);
}
if (meta && meta.notExists) {
return this._rm(id, name, options);
}
const metaID = this.getFileId(id, name, true);
const dataID = this.getFileId(id, name, false);
await this._delBinaryState(dataID);
await this.client.del(metaID);
}
unlink(id, name, options, callback) {
if (typeof options === 'function') {
callback = options;
options = null;
}
if (options?.acl) {
options.acl = null;
}
if (typeof name !== 'string') {
name = '';
}
if (name.startsWith('/')) {
name = name.substring(1);
}
this.checkFileRights(id, name, options, CONSTS.ACCESS_DELETE, async (err, options, meta) => {
if (err) {
return tools.maybeCallbackWithError(callback, err);
}
if (!options.acl.file.delete) {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_PERMISSION);
}
try {
const files = await this._unlink(id, name, options, meta);
return tools.maybeCallbackWithError(callback, null, files);
}
catch (e) {
return tools.maybeCallbackWithError(callback, e);
}
});
}
unlinkAsync(id, name, options) {
return new Promise((resolve, reject) => this.unlink(id, name, options, err => (err ? reject(err) : resolve())));
}
delFile(id, name, options, callback) {
return this.unlink(id, name, options, callback);
}
delFileAsync(id, name, options) {
return this.unlinkAsync(id, name, options);
}
async _readDir(id, name, options, callback) {
name = this.normalizeFilename(name);
if (!this.client) {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_DB_CLOSED);
}
if (id === '') {
// special case for "root"
const dirID = this.getFileId('*', '*');
let keys;
try {
keys = await this._getKeysViaScan(dirID);
}
catch (e) {
return tools.maybeCallbackWithRedisError(callback, e);
}
if (!this.client) {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_DB_CLOSED);
}
const result = [];
if (!keys || !keys.length) {
return tools.maybeCallbackWithError(callback, null, result);
}
let lastDir;
keys.sort().forEach(dir => {
dir = dir.substring(this.fileNamespaceL, dir.indexOf('$%$'));
if (dir !== lastDir) {
result.push({
file: dir,
stats: {},
isDir: true,
});
}
lastDir = dir;
});
return tools.maybeCallbackWithError(callback, null, result);
}
try {
await this.validateMetaObject(id);
}
catch (e) {
return tools.maybeCallbackWithRedisError(callback, e);
}
const dirID = this.getFileId(id, `${name}${name.length ? '/' : ''}*`);
let keys;
try {
keys = await this._getKeysViaScan(dirID);
}
catch (e) {
return tools.maybeCallbackWithRedisError(callback, e);
}
if (!this.client) {
return tools.maybeCallbackWithError(callback, ERRORS.ERROR_DB_CLOSED);
}
const start = dirID.indexOf('$%$') + 3;
const end = '$%$meta'.length;
const baseName = name + (name.length ? '/' : '');
const dirs = [];
const deepLevel = baseName.split('/').length;
if (!keys || !keys.length) {
return tools.maybeCallbackWithError(callback, null, []);
}
keys = keys.sort().filter(key => {
if (key.endsWith('$%$meta')) {
const parts = key.substr(start, key.length - end).split('/');
if (parts.length === deepLevel) {
return !key.includes('/_data.json$%$') && key !== '_data.json'; // sort out "virtual