azurite
Version:
An open source Azure Storage API compatible server
1,102 lines • 99 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
const fs_1 = require("fs");
const lokijs_1 = tslib_1.__importDefault(require("lokijs"));
const v4_1 = tslib_1.__importDefault(require("uuid/v4"));
const utils_1 = require("../../common/utils/utils");
const utils_2 = require("../../common/utils/utils");
const ReadConditionalHeadersValidator_1 = require("../conditions/ReadConditionalHeadersValidator");
const WriteConditionalHeadersValidator_1 = require("../conditions/WriteConditionalHeadersValidator");
const StorageErrorFactory_1 = tslib_1.__importDefault(require("../errors/StorageErrorFactory"));
const Models = tslib_1.__importStar(require("../generated/artifacts/models"));
const PageBlobRangesManager_1 = tslib_1.__importDefault(require("../handlers/PageBlobRangesManager"));
const BlobLeaseAdapter_1 = tslib_1.__importDefault(require("../lease/BlobLeaseAdapter"));
const BlobLeaseSyncer_1 = tslib_1.__importDefault(require("../lease/BlobLeaseSyncer"));
const BlobReadLeaseValidator_1 = tslib_1.__importDefault(require("../lease/BlobReadLeaseValidator"));
const BlobWriteLeaseSyncer_1 = tslib_1.__importDefault(require("../lease/BlobWriteLeaseSyncer"));
const BlobWriteLeaseValidator_1 = tslib_1.__importDefault(require("../lease/BlobWriteLeaseValidator"));
const ContainerDeleteLeaseValidator_1 = tslib_1.__importDefault(require("../lease/ContainerDeleteLeaseValidator"));
const ContainerLeaseAdapter_1 = tslib_1.__importDefault(require("../lease/ContainerLeaseAdapter"));
const ContainerLeaseSyncer_1 = tslib_1.__importDefault(require("../lease/ContainerLeaseSyncer"));
const ContainerReadLeaseValidator_1 = tslib_1.__importDefault(require("../lease/ContainerReadLeaseValidator"));
const LeaseFactory_1 = tslib_1.__importDefault(require("../lease/LeaseFactory"));
const constants_1 = require("../utils/constants");
const BlobReferredExtentsAsyncIterator_1 = tslib_1.__importDefault(require("./BlobReferredExtentsAsyncIterator"));
const PageWithDelimiter_1 = tslib_1.__importDefault(require("./PageWithDelimiter"));
const utils_3 = require("../utils/utils");
/**
* This is a metadata source implementation for blob based on loki DB.
*
* Notice that, following design is for emulator purpose only, and doesn't design for best performance.
* We may want to optimize the persistency layer performance in the future. Such as by distributing metadata
* into different collections, or make binary payload write as an append-only pattern.
*
* Loki DB includes following collections and documents:
*
* -- SERVICE_PROPERTIES_COLLECTION // Collection contains service properties
* // Default collection name is $SERVICES_COLLECTION$
* // Each document maps to 1 account blob service
* // Unique document properties: accountName
* -- CONTAINERS_COLLECTION // Collection contains all containers
* // Default collection name is $CONTAINERS_COLLECTION$
* // Each document maps to 1 container
* // Unique document properties: accountName, (container)name
* -- BLOBS_COLLECTION // Collection contains all blobs
* // Default collection name is $BLOBS_COLLECTION$
* // Each document maps to a blob
* // Unique document properties: accountName, containerName, (blob)name, snapshot
* -- BLOCKS_COLLECTION // Block blob blocks collection includes all UNCOMMITTED blocks
* // Unique document properties: accountName, containerName, blobName, name, isCommitted
*
* @export
* @class LokiBlobMetadataStore
*/
class LokiBlobMetadataStore {
constructor(lokiDBPath, inMemory) {
this.lokiDBPath = lokiDBPath;
this.initialized = false;
this.closed = true;
this.SERVICES_COLLECTION = "$SERVICES_COLLECTION$";
this.CONTAINERS_COLLECTION = "$CONTAINERS_COLLECTION$";
this.BLOBS_COLLECTION = "$BLOBS_COLLECTION$";
this.BLOCKS_COLLECTION = "$BLOCKS_COLLECTION$";
this.pageBlobRangesManager = new PageBlobRangesManager_1.default();
this.db = new lokijs_1.default(lokiDBPath, inMemory ? {
persistenceMethod: "memory"
} : {
persistenceMethod: "fs",
autosave: true,
autosaveInterval: 5000
});
}
isInitialized() {
return this.initialized;
}
isClosed() {
return this.closed;
}
async init() {
await new Promise((resolve, reject) => {
(0, fs_1.stat)(this.lokiDBPath, (statError, stats) => {
if (!statError) {
this.db.loadDatabase({}, (dbError) => {
if (dbError) {
reject(dbError);
}
else {
resolve();
}
});
}
else {
// when DB file doesn't exist, ignore the error because following will re-create the file
resolve();
}
});
});
// In loki DB implementation, these operations are all sync. Doesn't need an async lock
// Create service properties collection if not exists
let servicePropertiesColl = this.db.getCollection(this.SERVICES_COLLECTION);
if (servicePropertiesColl === null) {
servicePropertiesColl = this.db.addCollection(this.SERVICES_COLLECTION, {
unique: ["accountName"]
});
}
// Create containers collection if not exists
if (this.db.getCollection(this.CONTAINERS_COLLECTION) === null) {
this.db.addCollection(this.CONTAINERS_COLLECTION, {
// Optimization for indexing and searching
// https://rawgit.com/techfort/LokiJS/master/jsdoc/tutorial-Indexing%20and%20Query%20performance.html
indices: ["accountName", "name"]
}); // Optimize for find operation
}
// Create containers collection if not exists
if (this.db.getCollection(this.BLOBS_COLLECTION) === null) {
this.db.addCollection(this.BLOBS_COLLECTION, {
indices: ["accountName", "containerName", "name", "snapshot"] // Optimize for find operation
});
}
// Create blocks collection if not exists
if (this.db.getCollection(this.BLOCKS_COLLECTION) === null) {
this.db.addCollection(this.BLOCKS_COLLECTION, {
indices: ["accountName", "containerName", "blobName", "name"] // Optimize for find operation
});
}
await new Promise((resolve, reject) => {
this.db.saveDatabase((err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
this.initialized = true;
this.closed = false;
}
/**
* Close loki DB.
*
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async close() {
await new Promise((resolve, reject) => {
this.db.close((err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
});
this.closed = true;
}
/**
* Clean LokiBlobMetadataStore.
*
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async clean() {
if (this.isClosed()) {
await (0, utils_1.rimrafAsync)(this.lokiDBPath);
return;
}
throw new Error(`Cannot clean LokiBlobMetadataStore, it's not closed.`);
}
iteratorExtents() {
return new BlobReferredExtentsAsyncIterator_1.default(this);
}
/**
* Update blob service properties. Create service properties if not exists in persistency layer.
*
* TODO: Account's service property should be created when storage account is created or metadata
* storage initialization. This method should only be responsible for updating existing record.
* In this way, we can reduce one I/O call to get account properties.
*
* @param {ServicePropertiesModel} serviceProperties
* @returns {Promise<ServicePropertiesModel>} undefined properties will be ignored during properties setup
* @memberof LokiBlobMetadataStore
*/
async setServiceProperties(context, serviceProperties) {
const coll = this.db.getCollection(this.SERVICES_COLLECTION);
const doc = coll.by("accountName", serviceProperties.accountName);
if (doc) {
doc.cors =
serviceProperties.cors === undefined
? doc.cors
: serviceProperties.cors;
doc.hourMetrics =
serviceProperties.hourMetrics === undefined
? doc.hourMetrics
: serviceProperties.hourMetrics;
doc.logging =
serviceProperties.logging === undefined
? doc.logging
: serviceProperties.logging;
doc.minuteMetrics =
serviceProperties.minuteMetrics === undefined
? doc.minuteMetrics
: serviceProperties.minuteMetrics;
doc.defaultServiceVersion =
serviceProperties.defaultServiceVersion === undefined
? doc.defaultServiceVersion
: serviceProperties.defaultServiceVersion;
doc.deleteRetentionPolicy =
serviceProperties.deleteRetentionPolicy === undefined
? doc.deleteRetentionPolicy
: serviceProperties.deleteRetentionPolicy;
doc.staticWebsite =
serviceProperties.staticWebsite === undefined
? doc.staticWebsite
: serviceProperties.staticWebsite;
return coll.update(doc);
}
else {
return coll.insert(serviceProperties);
}
}
/**
* Get service properties for specific storage account.
*
* @param {string} account
* @returns {Promise<ServicePropertiesModel | undefined>}
* @memberof LokiBlobMetadataStore
*/
async getServiceProperties(context, account) {
const coll = this.db.getCollection(this.SERVICES_COLLECTION);
const doc = coll.by("accountName", account);
return doc ? doc : undefined;
}
/**
* List containers with query conditions specified.
*
* @param {string} account
* @param {string} [prefix=""]
* @param {number} [maxResults=5000]
* @param {string} [marker=""]
* @returns {(Promise<[ContainerModel[], string | undefined]>)}
* @memberof LokiBlobMetadataStore
*/
async listContainers(context, account, prefix = "", maxResults = constants_1.DEFAULT_LIST_CONTAINERS_MAX_RESULTS, marker = "") {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const query = prefix === ""
? { name: { $gt: marker }, accountName: account }
: {
name: { $regex: `^${this.escapeRegex(prefix)}`, $gt: marker },
accountName: account
};
// Workaround for loki which will ignore $gt when providing $regex
const query2 = { name: { $gt: marker } };
const docs = coll
.chain()
.find(query)
.find(query2)
.simplesort("name")
.limit(maxResults + 1)
.data();
if (docs.length <= maxResults) {
return [
docs.map((doc) => {
return LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context).sync(new ContainerLeaseSyncer_1.default(doc));
}),
undefined
];
}
else {
// In this case, the last item is the one we get in addition, should set the Marker before it.
const nextMarker = docs[docs.length - 2].name;
docs.pop();
return [
docs.map((doc) => {
return LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context).sync(new ContainerLeaseSyncer_1.default(doc));
}),
nextMarker
];
}
}
/**
* Create a container.
*
* @param {ContainerModel} container
* @returns {Promise<ContainerModel>}
* @memberof LokiBlobMetadataStore
*/
async createContainer(context, container) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = coll.findOne({
accountName: container.accountName,
name: container.name
});
if (doc) {
const requestId = context ? context.contextId : undefined;
throw StorageErrorFactory_1.default.getContainerAlreadyExists(requestId);
}
return coll.insert(container);
}
/**
* Get container properties.
*
* @param {string} account
* @param {string} container
* @param {Context} context
* @param {Models.LeaseAccessConditions} [leaseAccessConditions]
* @returns {Promise<GetContainerPropertiesResponse>}
* @memberof LokiBlobMetadataStore
*/
async getContainerProperties(context, account, container, leaseAccessConditions) {
const doc = await this.getContainerWithLeaseUpdated(account, container, context);
new ContainerReadLeaseValidator_1.default(leaseAccessConditions).validate(new ContainerLeaseAdapter_1.default(doc), context);
const res = {
name: container,
properties: doc.properties,
metadata: doc.metadata
};
return res;
}
/**
* Delete container item if exists from persistency layer.
*
* Loki based implementation will delete container documents from Containers collection,
* blob documents from Blobs collection, and blocks documents from Blocks collection immediately.
*
* Persisted extents data will be deleted by GC.
*
* @param {string} account
* @param {string} container
* @param {Context} context
* @param {Models.ContainerDeleteMethodOptionalParams} [options]
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async deleteContainer(context, account, container, options = {}) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainerWithLeaseUpdated(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
new ContainerDeleteLeaseValidator_1.default(options.leaseAccessConditions).validate(new ContainerLeaseAdapter_1.default(doc), context);
coll.remove(doc);
const blobColl = this.db.getCollection(this.BLOBS_COLLECTION);
blobColl.findAndRemove({
accountName: account,
containerName: container
});
const blockColl = this.db.getCollection(this.BLOCKS_COLLECTION);
blockColl.findAndRemove({
accountName: account,
containerName: container
});
}
/**
* Set container metadata.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {Date} lastModified
* @param {string} etag
* @param {IContainerMetadata} [metadata]
* @param {Models.LeaseAccessConditions} [leaseAccessConditions]
* @param {Models.ModifiedAccessConditions} [modifiedAccessConditions]
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async setContainerMetadata(context, account, container, lastModified, etag, metadata, leaseAccessConditions, modifiedAccessConditions) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainerWithLeaseUpdated(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
new ContainerReadLeaseValidator_1.default(leaseAccessConditions).validate(new ContainerLeaseAdapter_1.default(doc), context);
doc.properties.lastModified = lastModified;
doc.properties.etag = etag;
doc.metadata = metadata;
return coll.update(doc);
}
/**
* Get container access policy.
*
* @param {string} account
* @param {string} container
* @param {Context} context
* @param {Models.LeaseAccessConditions} [leaseAccessConditions]
* @returns {Promise<GetContainerAccessPolicyResponse>}
* @memberof LokiBlobMetadataStore
*/
async getContainerACL(context, account, container, leaseAccessConditions) {
const doc = await this.getContainerWithLeaseUpdated(account, container, context);
new ContainerReadLeaseValidator_1.default(leaseAccessConditions).validate(new ContainerLeaseAdapter_1.default(doc), context);
const res = {
properties: doc.properties,
containerAcl: doc.containerAcl
};
return res;
}
/**
* Set container access policy.
*
* @param {string} account
* @param {string} container
* @param {SetContainerAccessPolicyOptions} setAclModel
* @param {Context} context
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async setContainerACL(context, account, container, setAclModel) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainerWithLeaseUpdated(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, setAclModel.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
new ContainerReadLeaseValidator_1.default(setAclModel.leaseAccessConditions).validate(new ContainerLeaseAdapter_1.default(doc), context);
doc.properties.publicAccess = setAclModel.publicAccess;
doc.containerAcl = setAclModel.containerAcl;
doc.properties.lastModified = setAclModel.lastModified;
doc.properties.etag = setAclModel.etag;
return coll.update(doc);
}
/**
* Acquire container lease.
*
* @param {string} account
* @param {string} container
* @param {Models.ContainerAcquireLeaseOptionalParams} options
* @param {Context} context
* @returns {Promise<AcquireContainerLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async acquireContainerLease(context, account, container, options) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainer(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context)
.acquire(options.duration, options.proposedLeaseId)
.sync(new ContainerLeaseSyncer_1.default(doc));
coll.update(doc);
return { properties: doc.properties, leaseId: doc.leaseId };
}
/**
* Release container lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} leaseId
* @param {Models.ContainerReleaseLeaseOptionalParams} [options={}]
* @returns {Promise<ReleaseContainerLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async releaseContainerLease(context, account, container, leaseId, options = {}) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainer(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context)
.release(leaseId)
.sync(new ContainerLeaseSyncer_1.default(doc));
coll.update(doc);
return doc.properties;
}
/**
* Renew container lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} leaseId
* @param {Models.ContainerRenewLeaseOptionalParams} [options={}]
* @returns {Promise<RenewContainerLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async renewContainerLease(context, account, container, leaseId, options = {}) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainer(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context)
.renew(leaseId)
.sync(new ContainerLeaseSyncer_1.default(doc));
coll.update(doc);
return { properties: doc.properties, leaseId: doc.leaseId };
}
/**
* Break container lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {(number | undefined)} breakPeriod
* @param {Models.ContainerBreakLeaseOptionalParams} [options={}]
* @returns {Promise<BreakContainerLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async breakContainerLease(context, account, container, breakPeriod, options = {}) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainer(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context)
.break(breakPeriod)
.sync(new ContainerLeaseSyncer_1.default(doc));
const leaseTimeSeconds = doc.properties.leaseState === Models.LeaseStateType.Breaking &&
doc.leaseBreakTime
? Math.round((doc.leaseBreakTime.getTime() - context.startTime.getTime()) / 1000)
: 0;
coll.update(doc);
return { properties: doc.properties, leaseTime: leaseTimeSeconds };
}
/**
* Change container lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} leaseId
* @param {string} proposedLeaseId
* @param {Models.ContainerChangeLeaseOptionalParams} [options={}]
* @returns {Promise<ChangeContainerLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async changeContainerLease(context, account, container, leaseId, proposedLeaseId, options = {}) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = await this.getContainer(account, container, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getContainerNotFound(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new ContainerLeaseAdapter_1.default(doc), context)
.change(leaseId, proposedLeaseId)
.sync(new ContainerLeaseSyncer_1.default(doc));
coll.update(doc);
return { properties: doc.properties, leaseId: doc.leaseId };
}
/**
* Check the existence of a container.
*
* @param {string} account
* @param {string} container
* @param {Context} [context]
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async checkContainerExist(context, account, container) {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = coll.findOne({ accountName: account, name: container });
if (!doc) {
const requestId = context ? context.contextId : undefined;
throw StorageErrorFactory_1.default.getContainerNotFound(requestId);
}
}
async listBlobs(context, account, container, delimiter, blob, prefix = "", maxResults = constants_1.DEFAULT_LIST_BLOBS_MAX_RESULTS, marker = "", includeSnapshots, includeUncommittedBlobs) {
const query = {};
if (prefix !== "") {
query.name = { $regex: `^${this.escapeRegex(prefix)}` };
}
if (blob !== undefined) {
query.name = blob;
}
if (account !== undefined) {
query.accountName = account;
}
if (container !== undefined) {
query.containerName = container;
}
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const page = new PageWithDelimiter_1.default(maxResults, delimiter, prefix);
const readPage = async (offset) => {
return await coll
.chain()
.find(query)
.where((obj) => {
return obj.name > marker;
})
.where((obj) => {
return includeSnapshots ? true : obj.snapshot.length === 0;
})
.where((obj) => {
return includeUncommittedBlobs ? true : obj.isCommitted;
})
.sort((obj1, obj2) => {
if (obj1.name === obj2.name)
return 0;
if (obj1.name > obj2.name)
return 1;
return -1;
})
.offset(offset)
.limit(maxResults)
.data();
};
const nameItem = (item) => {
return item.name;
};
const [blobItems, blobPrefixes, nextMarker] = await page.fill(readPage, nameItem);
return [
blobItems.map((doc) => {
doc.properties.contentMD5 = this.restoreUint8Array(doc.properties.contentMD5);
return LeaseFactory_1.default.createLeaseState(new BlobLeaseAdapter_1.default(doc), context).sync(new BlobLeaseSyncer_1.default(doc));
}),
blobPrefixes,
nextMarker
];
}
async listAllBlobs(maxResults = constants_1.DEFAULT_LIST_BLOBS_MAX_RESULTS, marker = "", includeSnapshots, includeUncommittedBlobs) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const docs = await coll
.chain()
.where((obj) => {
return obj.name > marker;
})
.where((obj) => {
return includeSnapshots ? true : obj.snapshot.length === 0;
})
.where((obj) => {
return includeUncommittedBlobs ? true : obj.isCommitted;
})
.simplesort("name")
.limit(maxResults + 1)
.data();
for (const doc of docs) {
const blobDoc = doc;
blobDoc.properties.contentMD5 = this.restoreUint8Array(blobDoc.properties.contentMD5);
}
if (docs.length <= maxResults) {
return [docs, undefined];
}
else {
const nextMarker = docs[docs.length - 2].name;
docs.pop();
return [docs, nextMarker];
}
}
/**
* Create blob item in persistency layer. Will replace if blob exists.
*
* @param {Context} context
* @param {BlobModel} blob
* @param {Models.LeaseAccessConditions} [leaseAccessConditions]
* @param {Models.ModifiedAccessConditions} [modifiedAccessConditions]
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async createBlob(context, blob, leaseAccessConditions, modifiedAccessConditions) {
await this.checkContainerExist(context, blob.accountName, blob.containerName);
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const blobDoc = coll.findOne({
accountName: blob.accountName,
containerName: blob.containerName,
name: blob.name,
snapshot: blob.snapshot
});
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, modifiedAccessConditions, blobDoc);
// Create if not exists
if (modifiedAccessConditions &&
modifiedAccessConditions.ifNoneMatch === "*" &&
blobDoc) {
throw StorageErrorFactory_1.default.getBlobAlreadyExists(context.contextId);
}
if (blobDoc) {
LeaseFactory_1.default.createLeaseState(new BlobLeaseAdapter_1.default(blobDoc), context)
.validate(new BlobWriteLeaseValidator_1.default(leaseAccessConditions))
.sync(new BlobLeaseSyncer_1.default(blob)); // Keep original blob lease
if (blobDoc.properties !== undefined &&
blobDoc.properties.accessTier === Models.AccessTier.Archive) {
throw StorageErrorFactory_1.default.getBlobArchived(context.contextId);
}
coll.remove(blobDoc);
}
delete blob.$loki;
return coll.insert(blob);
}
/**
* Create snapshot.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided
* @returns {Promise<CreateSnapshotResponse>}
* @memberof LokiBlobMetadataStore
*/
async createSnapshot(context, account, container, blob, leaseAccessConditions, metadata, modifiedAccessConditions) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false, true);
(0, ReadConditionalHeadersValidator_1.validateReadConditions)(context, modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
new BlobReadLeaseValidator_1.default(leaseAccessConditions).validate(new BlobLeaseAdapter_1.default(doc), context);
const snapshotTime = (0, utils_1.convertDateTimeStringMsTo7Digital)(context.startTime.toISOString());
const snapshotBlob = {
name: doc.name,
deleted: false,
snapshot: snapshotTime,
properties: Object.assign({}, doc.properties),
metadata: metadata ? Object.assign({}, metadata) : Object.assign({}, doc.metadata),
blobTags: doc.blobTags,
accountName: doc.accountName,
containerName: doc.containerName,
pageRangesInOrder: doc.pageRangesInOrder === undefined
? undefined
: doc.pageRangesInOrder.slice(),
isCommitted: doc.isCommitted,
committedBlocksInOrder: doc.committedBlocksInOrder === undefined
? undefined
: doc.committedBlocksInOrder.slice(),
persistency: doc.persistency === undefined ? undefined : Object.assign({}, doc.persistency)
};
new BlobLeaseSyncer_1.default(snapshotBlob).sync({
leaseId: undefined,
leaseExpireTime: undefined,
leaseDurationSeconds: undefined,
leaseBreakTime: undefined,
leaseDurationType: undefined,
leaseState: undefined,
leaseStatus: undefined
});
coll.insert(snapshotBlob);
return {
properties: snapshotBlob.properties,
snapshot: snapshotTime
};
}
/**
* Gets a blob item from persistency layer by container name and blob name.
* Will return block list or page list as well for downloading.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {string} [snapshot=""]
* @param {Models.LeaseAccessConditions} [leaseAccessConditions]
* @param {Models.ModifiedAccessConditions} [modifiedAccessConditions]
* @returns {Promise<BlobModel>}
* @memberof LokiBlobMetadataStore
*/
async downloadBlob(context, account, container, blob, snapshot = "", leaseAccessConditions, modifiedAccessConditions) {
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, snapshot, context, false, true);
(0, ReadConditionalHeadersValidator_1.validateReadConditions)(context, modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
new BlobReadLeaseValidator_1.default(leaseAccessConditions).validate(new BlobLeaseAdapter_1.default(doc), context);
return doc;
}
/**
* Gets a blob item from persistency layer by container name and blob name.
* Will return block list or page list as well for downloading.
*
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {string} [snapshot]
* @returns {(Promise<BlobModel | undefined>)}
* @memberof LokiBlobMetadataStore
*/
async getBlob(context, account, container, blob, snapshot = "") {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const blobDoc = coll.findOne({
accountName: account,
containerName: container,
name: blob,
snapshot
});
if (blobDoc) {
const blobModel = blobDoc;
blobModel.properties.contentMD5 = this.restoreUint8Array(blobModel.properties.contentMD5);
return blobDoc;
}
else {
return undefined;
}
}
/**
* Get blob properties.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {string} [snapshot=""]
* @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions
* @param {Models.ModifiedAccessConditions} [modifiedAccessConditions]
* @returns {Promise<GetBlobPropertiesRes>}
* @memberof LokiBlobMetadataStore
*/
async getBlobProperties(context, account, container, blob, snapshot = "", leaseAccessConditions, modifiedAccessConditions) {
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, snapshot, context, false, true);
(0, ReadConditionalHeadersValidator_1.validateReadConditions)(context, modifiedAccessConditions, doc);
// When block blob don't have commited block, should return 404
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
new BlobReadLeaseValidator_1.default(leaseAccessConditions).validate(new BlobLeaseAdapter_1.default(doc), context);
doc.properties.tagCount = (0, utils_3.getBlobTagsCount)(doc.blobTags);
return {
properties: doc.properties,
metadata: doc.metadata,
blobCommittedBlockCount: doc.properties.blobType === Models.BlobType.AppendBlob
? (doc.committedBlocksInOrder || []).length
: undefined
};
}
/**
* Delete blob or its snapshots.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {Models.BlobDeleteMethodOptionalParams} options
* @returns {Promise<void>}
* @memberof LokiBlobMetadataStore
*/
async deleteBlob(context, account, container, blob, options) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
await this.checkContainerExist(context, account, container);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, options.snapshot, context, false);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
const againstBaseBlob = doc.snapshot === "";
// Check bad requests
if (!againstBaseBlob && options.deleteSnapshots !== undefined) {
throw StorageErrorFactory_1.default.getInvalidOperation(context.contextId, "Invalid operation against a blob snapshot.");
}
new BlobWriteLeaseValidator_1.default(options.leaseAccessConditions).validate(new BlobLeaseAdapter_1.default(doc), context);
// Scenario: Delete base blob only
if (againstBaseBlob && options.deleteSnapshots === undefined) {
const count = coll.count({
accountName: account,
containerName: container,
name: blob
});
if (count > 1) {
throw StorageErrorFactory_1.default.getSnapshotsPresent(context.contextId);
}
else {
coll.findAndRemove({
accountName: account,
containerName: container,
name: blob
});
}
}
// Scenario: Delete one snapshot only
if (!againstBaseBlob) {
coll.findAndRemove({
accountName: account,
containerName: container,
name: blob,
snapshot: doc.snapshot
});
}
// Scenario: Delete base blob and snapshots
if (againstBaseBlob &&
options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Include) {
coll.findAndRemove({
accountName: account,
containerName: container,
name: blob
});
}
// Scenario: Delete all snapshots only
if (againstBaseBlob &&
options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Only) {
const query = {
accountName: account,
containerName: container,
name: blob,
snapshot: { $gt: "" }
};
coll.findAndRemove(query);
}
}
/**
* Set blob HTTP headers.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions
* @param {(Models.BlobHTTPHeaders | undefined)} blobHTTPHeaders
* @param {Models.ModifiedAccessConditions} [modifiedAccessConditions]
* @returns {Promise<Models.BlobProperties>}
* @memberof LokiBlobMetadataStore
*/
async setBlobHTTPHeaders(context, account, container, blob, leaseAccessConditions, blobHTTPHeaders, modifiedAccessConditions) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false, true);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
const lease = new BlobLeaseAdapter_1.default(doc);
new BlobWriteLeaseValidator_1.default(leaseAccessConditions).validate(lease, context);
const blobHeaders = blobHTTPHeaders;
const blobProps = doc.properties;
// as per https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties#remarks
// If any one or more of the following properties is set in the request,
// then all of these properties are set together.
// If a value is not provided for a given property when at least one
// of the properties listed below is set, then that property will
// be cleared for the blob.
if (blobHeaders !== undefined) {
blobProps.cacheControl = blobHeaders.blobCacheControl;
blobProps.contentType = blobHeaders.blobContentType;
blobProps.contentMD5 = blobHeaders.blobContentMD5;
blobProps.contentEncoding = blobHeaders.blobContentEncoding;
blobProps.contentLanguage = blobHeaders.blobContentLanguage;
blobProps.contentDisposition = blobHeaders.blobContentDisposition;
}
doc.properties = blobProps;
doc.properties.etag = (0, utils_2.newEtag)();
blobProps.lastModified = context.startTime ? context.startTime : new Date();
new BlobWriteLeaseSyncer_1.default(doc).sync(lease);
coll.update(doc);
return doc.properties;
}
/**
* Set blob metadata.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions
* @param {(Models.BlobMetadata | undefined)} metadata
* @param {Models.ModifiedAccessConditions} [modifiedAccessConditions]
* @returns {Promise<Models.BlobProperties>}
* @memberof LokiBlobMetadataStore
*/
async setBlobMetadata(context, account, container, blob, leaseAccessConditions, metadata, modifiedAccessConditions) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false, true);
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, modifiedAccessConditions, doc);
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
const lease = new BlobLeaseAdapter_1.default(doc);
new BlobWriteLeaseValidator_1.default(leaseAccessConditions).validate(lease, context);
new BlobWriteLeaseSyncer_1.default(doc).sync(lease);
doc.metadata = metadata;
doc.properties.etag = (0, utils_2.newEtag)();
doc.properties.lastModified = context.startTime || new Date();
coll.update(doc);
return doc.properties;
}
/**
* Acquire blob lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {number} duration
* @param {string} [proposedLeaseId]
* @param {Models.BlobAcquireLeaseOptionalParams} [options={}]
* @returns {Promise<AcquireBlobLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async acquireBlobLease(context, account, container, blob, duration, proposedLeaseId, options = {}) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false); // This may return an uncommitted blob, or undefined for an unexist blob
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
// Azure Storage allows lease for a uncommitted blob
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
if (doc.snapshot !== "") {
throw StorageErrorFactory_1.default.getBlobSnapshotsPresent(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new BlobLeaseAdapter_1.default(doc), context)
.acquire(duration, proposedLeaseId)
.sync(new BlobLeaseSyncer_1.default(doc));
coll.update(doc);
return { properties: doc.properties, leaseId: doc.leaseId };
}
/**
* Release blob.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {string} leaseId
* @param {Models.BlobReleaseLeaseOptionalParams} [options={}]
* @returns {Promise<ReleaseBlobLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async releaseBlobLease(context, account, container, blob, leaseId, options = {}) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false); // This may return an uncommitted blob, or undefined for an unexist blob
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
// Azure Storage allows lease for a uncommitted blob
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
if (doc.snapshot !== "") {
throw StorageErrorFactory_1.default.getBlobSnapshotsPresent(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new BlobLeaseAdapter_1.default(doc), context)
.release(leaseId)
.sync(new BlobLeaseSyncer_1.default(doc));
coll.update(doc);
return doc.properties;
}
/**
* Renew blob lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {string} leaseId
* @param {Models.BlobRenewLeaseOptionalParams} [options={}]
* @returns {Promise<RenewBlobLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async renewBlobLease(context, account, container, blob, leaseId, options = {}) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false); // This may return an uncommitted blob, or undefined for an unexist blob
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
// Azure Storage allows lease for a uncommitted blob
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
if (doc.snapshot !== "") {
throw StorageErrorFactory_1.default.getBlobSnapshotsPresent(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new BlobLeaseAdapter_1.default(doc), context)
.renew(leaseId)
.sync(new BlobLeaseSyncer_1.default(doc));
coll.update(doc);
return { properties: doc.properties, leaseId: doc.leaseId };
}
/**
* Change blob lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {string} leaseId
* @param {string} proposedLeaseId
* @param {Models.BlobChangeLeaseOptionalParams} [option={}]
* @returns {Promise<ChangeBlobLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async changeBlobLease(context, account, container, blob, leaseId, proposedLeaseId, options = {}) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false); // This may return an uncommitted blob, or undefined for an unexist blob
(0, WriteConditionalHeadersValidator_1.validateWriteConditions)(context, options.modifiedAccessConditions, doc);
// Azure Storage allows lease for a uncommitted blob
if (!doc) {
throw StorageErrorFactory_1.default.getBlobNotFound(context.contextId);
}
if (doc.snapshot !== "") {
throw StorageErrorFactory_1.default.getBlobSnapshotsPresent(context.contextId);
}
LeaseFactory_1.default.createLeaseState(new BlobLeaseAdapter_1.default(doc), context)
.change(leaseId, proposedLeaseId)
.sync(new BlobLeaseSyncer_1.default(doc));
coll.update(doc);
return { properties: doc.properties, leaseId: doc.leaseId };
}
/**
* Break blob lease.
*
* @param {Context} context
* @param {string} account
* @param {string} container
* @param {string} blob
* @param {(number | undefined)} breakPeriod
* @param {Models.BlobBreakLeaseOptionalParams} [options={}]
* @returns {Promise<BreakBlobLeaseResponse>}
* @memberof LokiBlobMetadataStore
*/
async breakBlobLease(context, account, container, blob, breakPeriod, options = {}) {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = await this.getBlobWithLeaseUpdated(account, container, blob, undefined, context, false); // This may return an uncommitted blob, or undefined for an unexist blob
(0, WriteConditionalHeadersValidator_1.validateWri