azurite
Version:
An open source Azure Storage API compatible server
705 lines • 26.6 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
const fs_1 = require("fs");
const lokijs_1 = tslib_1.__importDefault(require("lokijs"));
const StorageErrorFactory_1 = tslib_1.__importDefault(require("../errors/StorageErrorFactory"));
const QueueReferredExtentsAsyncIterator_1 = tslib_1.__importDefault(require("./QueueReferredExtentsAsyncIterator"));
const utils_1 = require("../../common/utils/utils");
/**
* This is a metadata source implementation for queue based on loki DB.
*
* Notice that, following design is for emulator purpose only, and doesn't design for best performance.
* We may want to optimize the persistency layer performance in the future. Such as by distributing metadata
* into different collections, or make binary payload write as an append-only pattern.
*
* Loki DB includes following collections and documents:
*
* -- SERVICE_PROPERTIES_COLLECTION // Collection contains service properties
* // Default collection name is $SERVICES_COLLECTION$
* // Each document maps to 1 account queue service
* // Unique document properties: accountName
* -- QUEUES_COLLECTION // Collection contains all queues
* // Default collection name is $QUEUES_COLLECTION$
* // Each document maps to 1 queue
* // Unique document properties: accountName, (queue)name
* -- MESSAGES_COLLECTION // Collection contains all messages
* // Default collection name is $MESSAGES_COLLECTION$
* // Each document maps to a message
* // Unique document properties: accountName, queueName, messageId
*
* @export
* @class LokiQueueMetadataStore
*/
class LokiQueueMetadataStore {
constructor(lokiDBPath, inMemory) {
this.lokiDBPath = lokiDBPath;
this.initialized = false;
this.closed = false;
this.SERVICES_COLLECTION = "$SERVICES_COLLECTION$";
this.QUEUES_COLLECTION = "$QUEUES_COLLECTION$";
this.MESSAGES_COLLECTION = "$MESSAGES_COLLECTION$";
this.db = new lokijs_1.default(lokiDBPath, inMemory ? {
persistenceMethod: "memory"
} : {
persistenceMethod: "fs",
autosave: true,
autosaveInterval: 5000
});
}
isInitialized() {
return this.initialized;
}
isClosed() {
return this.closed;
}
async init() {
await new Promise((resolve, reject) => {
(0, fs_1.stat)(this.lokiDBPath, (statError, stats) => {
if (!statError) {
this.db.loadDatabase({}, (dbError) => {
if (dbError) {
reject(dbError);
}
else {
resolve();
}
});
}
else {
// when DB file doesn't exist, ignore the error because following will re-create the file
resolve();
}
});
});
// In loki DB implementation, these operations are all sync. Doesn't need an async lock
// Create service properties collection if not exists
let servicePropertiesColl = this.db.getCollection(this.SERVICES_COLLECTION);
if (servicePropertiesColl === null) {
servicePropertiesColl = this.db.addCollection(this.SERVICES_COLLECTION, {
unique: ["accountName"]
});
}
// Create queues collection if not exists
if (this.db.getCollection(this.QUEUES_COLLECTION) === null) {
this.db.addCollection(this.QUEUES_COLLECTION, {
// Optimization for indexing and searching
// https://rawgit.com/techfort/LokiJS/master/jsdoc/tutorial-Indexing%20and%20Query%20performance.html
indices: ["accountName", "name"]
}); // Optimize for find operation
}
// Create messages collection if not exists
if (this.db.getCollection(this.MESSAGES_COLLECTION) === null) {
this.db.addCollection(this.MESSAGES_COLLECTION, {
indices: ["accountName", "queueName", "messageId", "visibleTime"] // Optimize for find operation
});
}
await new Promise((resolve, reject) => {
this.db.saveDatabase((err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
this.initialized = true;
this.closed = false;
}
/**
* Close loki DB.
*
* @returns {Promise<void>}
* @memberof LokiQueueDataStore
*/
async close() {
await new Promise((resolve, reject) => {
this.db.close((err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
this.closed = true;
}
/**
* Clean LokiQueueMetadataStore.
*
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async clean() {
if (this.isClosed()) {
await (0, utils_1.rimrafAsync)(this.lokiDBPath);
return;
}
throw new Error(`Cannot clean LokiQueueMetadataStore, it's not closed.`);
}
/**
* Update queue service properties. Create service properties document if not exists in persistency layer.
* Assume service properties collection has been created during start method.
*
* @param {ServicePropertiesModel} updateProperties
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async updateServiceProperties(updateProperties) {
const coll = this.db.getCollection(this.SERVICES_COLLECTION);
const doc = coll.by("accountName", updateProperties.accountName);
if (doc) {
doc.cors =
updateProperties.cors === undefined ? doc.cors : updateProperties.cors;
doc.hourMetrics =
updateProperties.hourMetrics === undefined
? doc.hourMetrics
: updateProperties.hourMetrics;
doc.logging =
updateProperties.logging === undefined
? doc.logging
: updateProperties.logging;
doc.minuteMetrics =
updateProperties.minuteMetrics === undefined
? doc.minuteMetrics
: updateProperties.minuteMetrics;
coll.update(doc);
}
else {
coll.insert(updateProperties);
}
}
/**
* Get service properties for specific storage account.
*
* @template T
* @param {string} account
* @returns {(Promise<ServicePropertiesModel | undefined>)}
* @memberof LokiQueueMetadataStore
*/
async getServiceProperties(account) {
const coll = this.db.getCollection(this.SERVICES_COLLECTION);
const doc = coll.by("accountName", account);
return doc ? doc : undefined;
}
/**
* List queues with query conditions specified.
*
* @template T
* @param {string} account
* @param {string} [prefix=""]
* @param {number} [maxResults=5000]
* @param {number} [marker=0]
* @returns {(Promise<[QueueModel[], number | undefined]>)} A tuple including queues and next marker
* @memberof LokiQueueMetadataStore
*/
async listQueues(account, prefix = "", maxResults = 5000, marker = 0) {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const query = prefix === ""
? { $loki: { $gt: marker }, accountName: account }
: {
name: { $regex: `^${this.escapeRegex(prefix)}` },
$loki: { $gt: marker },
accountName: account
};
// Get one more item to help check if the query reach the tail of the collection.
const docs = coll
.chain()
.find(query)
.sort((obj1, obj2) => {
if (obj1.name === obj2.name)
return 0;
if (obj1.name > obj2.name)
return 1;
return -1;
})
.limit(maxResults + 1)
.data();
const queues = [];
for (let i = 0; i < maxResults && i < docs.length; i++) {
queues.push(this.queueCopy(docs[i]));
}
if (docs.length <= maxResults) {
return [queues, undefined];
}
else {
// In this case, the last item is the one we get in addition, should set the Marker before it.
const nextMarker = docs[docs.length - 1].$loki - 1;
return [queues, nextMarker];
}
}
/**
* Get a queue item from persistency layer by account and queue name.
*
* @param {string} account
* @param {string} queue
* @param {Context} [context]
* @returns {Promise<QueueModel>}
* @memberof LokiQueueMetadataStore
*/
async getQueue(account, queue, context) {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getQueueNotFound(requestId);
}
return doc;
}
/**
* Create a queue in persistency layer.
* Return 201 if create a new one, 204 if a same one exist, 409 error if a conflicting one exist.
* @see https://docs.microsoft.com/en-us/rest/api/storageservices/create-queue4
*
* @param {QueueModel} queue
* @param {Context} [context]
* @returns {Promise<QUEUE_STATUSCODE>}
* @memberof LokiQueueMetadataStore
*/
async createQueue(queue, context) {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({
accountName: queue.accountName,
name: queue.name
});
// Check whether a conflict exists if there exist a queue with the given name.
// If the exist queue has the same metadata as the given queue, then return 204, else throw 409 error.
if (doc) {
const docMeta = doc.metadata;
const queueMeta = queue.metadata;
// Check if both metadata is empty.
if (queueMeta === undefined) {
if (docMeta !== undefined) {
throw StorageErrorFactory_1.default.getQueueAlreadyExists(context ? context.contextID : undefined);
}
else {
return 204;
}
}
if (docMeta === undefined) {
throw StorageErrorFactory_1.default.getQueueAlreadyExists(context ? context.contextID : undefined);
}
// Check if the numbers of metadata are equal.
if (Object.keys(queueMeta).length !== Object.keys(docMeta).length) {
throw StorageErrorFactory_1.default.getQueueAlreadyExists(context ? context.contextID : undefined);
}
const nameMap = new Map();
for (const item in queueMeta) {
if (queueMeta.hasOwnProperty(item)) {
nameMap.set(item.toLowerCase(), item);
}
}
// Check if all the metadata of exist queue is the same as another.
for (const item in docMeta) {
if (docMeta.hasOwnProperty(item)) {
const queueMetaName = nameMap.get(item.toLowerCase());
if (queueMetaName === undefined) {
throw StorageErrorFactory_1.default.getQueueAlreadyExists(context ? context.contextID : undefined);
}
if (docMeta[item] !== queueMeta[queueMetaName]) {
throw StorageErrorFactory_1.default.getQueueAlreadyExists(context ? context.contextID : undefined);
}
}
}
return 204;
}
coll.insert(queue);
return 201;
}
/**
* Delete a queue and its all messages.
*
* @param {string} account
* @param {string} queue
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async deleteQueue(account, queue, context) {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getQueueNotFound(requestId);
}
coll.remove(doc);
const messageColl = this.db.getCollection(this.MESSAGES_COLLECTION);
messageColl.findAndRemove({
accountName: account,
queueName: queue
});
}
/**
* Update the ACL of an exist queue item in persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {QueueACL} [queueACL]
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async setQueueACL(account, queue, queueACL, context) {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getQueueNotFound(requestId);
}
doc.queueAcl = queueACL;
coll.update(doc);
}
/**
* Update the metadata of an exist queue item in persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {QueueMetadata} [metadata]
* @param {string} [requestId]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async setQueueMetadata(account, queue, metadata, context) {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getQueueNotFound(requestId);
}
doc.metadata = metadata;
coll.update(doc);
}
/**
* Get the number of messages of a queue from persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {string} [requestId]
* @returns {number}
* @memberof LokiQueueMetadataStore
*/
async getMessagesCount(account, queue, context) {
const queueColl = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = queueColl.findOne({ accountName: account, name: queue });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getQueueNotFound(requestId);
}
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const query = { accountName: account, queueName: queue };
const numberOfMessages = coll.count(query);
return numberOfMessages;
}
/**
* Insert a message in metadata.
* The existence of the certain queue should be validated before.
*
* @param {MessageModel} message
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async insertMessage(message, context) {
this.checkQueueExist(message.accountName, message.queueName, context);
const saveMessage = this.messageCopy(message);
saveMessage.timeNextVisible = saveMessage.timeNextVisible.getTime();
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
coll.insert(saveMessage);
}
/**
* peek messages of a given number in persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {number} [numOfMessages]
* @param {Date} [queryDate]
* @param {Context} [context]
* @returns {Promise<MessageModel[]>}
* @memberof LokiQueueMetadataStore
*/
async peekMessages(account, queue, numOfMessages, queryDate, context) {
this.checkQueueExist(account, queue, context);
this.clearExpiredMessages(account, queue, context);
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const queryTime = queryDate ? queryDate.getTime() : new Date().getTime();
const query = {
accountName: account,
queueName: queue,
timeNextVisible: { $lte: queryTime }
};
if (numOfMessages === undefined) {
numOfMessages = 1;
}
const docs = coll
.chain()
.find(query)
.compoundsort([
["timeNextVisible", false],
["$loki", false]
])
.limit(numOfMessages)
.data();
const res = [];
for (const doc of docs) {
doc.timeNextVisible = new Date(Number(doc.timeNextVisible));
res.push(this.messageCopy(doc));
doc.timeNextVisible = doc.timeNextVisible.getTime();
}
return res;
}
/**
* Dequeue messages from a queue in persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {Date} timeNextVisible
* @param {string} popReceipt
* @param {number} [numOfMessages]
* @param {Date} [queryDate]
* @param {Context} [context]
* @returns {Promise<MessageModel[]>}
* @memberof LokiQueueMetadataStore
*/
async getMessages(account, queue, timeNextVisible, popReceipt, numOfMessages, queryDate, context) {
this.checkQueueExist(account, queue, context);
this.clearExpiredMessages(account, queue, context);
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const queryTime = queryDate ? queryDate.getTime() : new Date().getTime();
const query = {
accountName: account,
queueName: queue,
timeNextVisible: { $lte: queryTime }
};
if (numOfMessages === undefined || numOfMessages < 1) {
numOfMessages = 1;
}
const docs = coll
.chain()
.find(query)
.compoundsort([
["timeNextVisible", false],
["$loki", false]
])
.limit(numOfMessages)
.data();
const visibleTimeInMillisecond = timeNextVisible.getTime();
for (const doc of docs) {
doc.timeNextVisible = visibleTimeInMillisecond;
doc.popReceipt = popReceipt;
doc.dequeueCount = Number(doc.dequeueCount) + 1;
}
coll.update(docs);
const res = [];
for (const doc of docs) {
doc.timeNextVisible = timeNextVisible;
res.push(this.messageCopy(doc));
doc.timeNextVisible = visibleTimeInMillisecond;
}
return res;
}
/**
* Delete the metadata of a message from persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {string} messageId
* @param {string} validatingPopReceipt
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async deleteMessage(account, queue, messageId, validatingPopReceipt, context) {
this.checkQueueExist(account, queue, context);
this.clearExpiredMessages(account, queue, context);
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const doc = coll.findOne({
accountName: account,
queueName: queue,
messageId
});
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getMessageNotFound(requestId);
}
if (doc.popReceipt !== validatingPopReceipt) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getPopReceiptMismatch(requestId);
}
coll.remove(doc);
}
/**
* Update the metadata of an exist message in persistency layer.
*
* @param {MessageUpdateProperties} message
* @param {string} validatingPopReceipt
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async updateMessage(message, validatingPopReceipt, context) {
this.checkQueueExist(message.accountName, message.queueName, context);
this.clearExpiredMessages(message.accountName, message.queueName, context);
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const doc = coll.findOne({
accountName: message.accountName,
queueName: message.queueName,
messageId: message.messageId
});
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getMessageNotFound(requestId);
}
if (doc.popReceipt !== validatingPopReceipt) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getPopReceiptMismatch(requestId);
}
doc.popReceipt = message.popReceipt;
doc.timeNextVisible = message.timeNextVisible.getTime();
if (message.persistency !== undefined) {
doc.persistency = message.persistency;
}
coll.update(doc);
}
/**
* Clear the metadata of all messages in a given queue from persistency layer.
*
* @param {string} account
* @param {string} queue
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async clearMessages(account, queue, context) {
this.checkQueueExist(account, queue, context);
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
coll.findAndRemove({
accountName: account,
queueName: queue
});
}
/**
* List messages.
*
* @param {number} [maxResults]
* @param {(number | undefined)} [marker]
* @returns {(Promise<[MessageModel[], number | undefined]>)}
* @memberof IQueueMetadataStore
*/
async listMessages(maxResults, marker) {
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const query = {
$loki: { $gt: marker }
};
if (maxResults === undefined) {
maxResults = 5000;
}
const docs = coll.chain().find(query).limit(maxResults).data();
if (docs.length < maxResults) {
return [docs, undefined];
}
const nextMarker = docs[docs.length - 1].$loki;
return [docs, nextMarker];
}
iteratorExtents() {
return new QueueReferredExtentsAsyncIterator_1.default(this);
}
/**
* Check the existence of a queue.
*
* @private
* @param {string} account
* @param {string} queue
* @param {Context} [context]
* @memberof LokiQueueMetadataStore
*/
checkQueueExist(account, queue, context) {
const queueColl = this.db.getCollection(this.QUEUES_COLLECTION);
const queueDoc = queueColl.findOne({ accountName: account, name: queue });
if (!queueDoc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory_1.default.getQueueNotFound(requestId);
}
}
/**
* Deep copy a message, return the new object.
*
* @private
* @param {MessageModel} message
* @returns {MessageModel}
* @memberof LokiQueueMetadataStore
*/
messageCopy(message) {
const copyMessage = {
accountName: message.accountName,
queueName: message.queueName,
messageId: message.messageId,
popReceipt: message.popReceipt,
timeNextVisible: new Date(message.timeNextVisible),
persistency: {
id: message.persistency.id,
offset: message.persistency.offset,
count: message.persistency.count
},
insertionTime: new Date(message.insertionTime),
expirationTime: new Date(message.expirationTime),
dequeueCount: message.dequeueCount
};
return copyMessage;
}
/**
* Deep copy a queue, return the new object.
*
* @private
* @param {QueueModel} queue
* @returns {QueueModel}
* @memberof LokiQueueMetadataStore
*/
queueCopy(queue) {
const copyQueue = {
accountName: queue.accountName,
name: queue.name,
queueAcl: queue.queueAcl,
metadata: queue.metadata
};
return copyQueue;
}
/**
* Escape a string to be used as a regex.
*
* @private
* @param {string} regex
* @returns {string}
* @memberof LokiQueueMetadataStore
*/
escapeRegex(regex) {
return regex.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&");
}
/**
* Clean the expired messages.
*
* @param {string} account
* @param {string} queue
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiQueueMetadataStore
*/
async clearExpiredMessages(account, queue, context) {
this.checkQueueExist(account, queue, context);
const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const queryTime = new Date().getTime();
const query = {
accountName: account,
queueName: queue,
expirationTime: { $lte: queryTime }
};
const docs = coll
.chain()
.find(query)
.data();
coll.remove(docs);
}
}
exports.default = LokiQueueMetadataStore;
//# sourceMappingURL=LokiQueueMetadataStore.js.map