azurite
Version:
An open source Azure Storage API compatible server
290 lines • 11.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.SharedChunkStore = exports.DEFAULT_EXTENT_MEMORY_LIMIT = exports.MemoryExtentChunkStore = void 0;
const tslib_1 = require("tslib");
const IBlobMetadataStore_1 = require("../../blob/persistence/IBlobMetadataStore");
const ZeroBytesStream_1 = tslib_1.__importDefault(require("../ZeroBytesStream"));
const uuid = require("uuid");
const multistream = require("multistream");
const stream_1 = require("stream");
const os_1 = require("os");
class MemoryExtentChunkStore {
constructor(sizeLimit) {
this._chunks = new Map();
this._totalSize = 0;
this._sizeLimit = sizeLimit;
}
clear(categoryName) {
let category = this._chunks.get(categoryName);
if (!category) {
return;
}
this._totalSize -= category.totalSize;
this._chunks.delete(categoryName);
}
set(categoryName, chunk) {
if (!this.trySet(categoryName, chunk)) {
throw new Error(`Cannot add an extent chunk to the in-memory store. Size limit of ${this._sizeLimit} bytes will be exceeded.`);
}
}
trySet(categoryName, chunk) {
let category = this._chunks.get(categoryName);
if (!category) {
category = {
chunks: new Map(),
totalSize: 0
};
this._chunks.set(categoryName, category);
}
let delta = chunk.count;
const existing = category.chunks.get(chunk.id);
if (existing) {
delta -= existing.count;
}
if (this._sizeLimit != undefined && this._totalSize + delta > this._sizeLimit) {
return false;
}
category.chunks.set(chunk.id, chunk);
category.totalSize += delta;
this._totalSize += delta;
return true;
}
get(categoryName, id) {
return this._chunks.get(categoryName)?.chunks.get(id);
}
delete(categoryName, id) {
const category = this._chunks.get(categoryName);
if (!category) {
return false;
}
const existing = category.chunks.get(id);
if (!existing) {
return false;
}
category.chunks.delete(id);
category.totalSize -= existing.count;
this._totalSize -= existing.count;
if (category.chunks.size === 0) {
this._chunks.delete(categoryName);
}
return true;
}
totalSize() {
return this._totalSize;
}
setSizeLimit(sizeLimit) {
if (sizeLimit && sizeLimit < this._totalSize) {
return false;
}
this._sizeLimit = sizeLimit;
return true;
}
sizeLimit() {
return this._sizeLimit;
}
}
exports.MemoryExtentChunkStore = MemoryExtentChunkStore;
// By default, allow up to half of the total memory to be used for in-memory
// extents. We don't use freemem (free memory instead of total memory) since
// that would lead to a decent amount of unpredictability.
exports.DEFAULT_EXTENT_MEMORY_LIMIT = Math.trunc((0, os_1.totalmem)() * 0.5);
exports.SharedChunkStore = new MemoryExtentChunkStore(exports.DEFAULT_EXTENT_MEMORY_LIMIT);
class MemoryExtentStore {
constructor(categoryName, chunks, metadata, logger, makeError) {
this.initialized = false;
this.closed = true;
this.categoryName = categoryName;
this.chunks = chunks;
this.metadataStore = metadata;
this.logger = logger;
this.makeError = makeError;
}
isInitialized() {
return this.initialized;
}
isClosed() {
return this.closed;
}
async init() {
if (!this.metadataStore.isInitialized()) {
await this.metadataStore.init();
}
this.initialized = true;
this.closed = false;
}
async close() {
if (!this.metadataStore.isClosed()) {
await this.metadataStore.close();
}
this.closed = true;
}
async clean() {
if (this.isClosed()) {
this.chunks.clear(this.categoryName);
return;
}
throw new Error(`Cannot clean MemoryExtentStore, it's not closed.`);
}
async appendExtent(data, contextId) {
const chunks = [];
let count = 0;
if (data instanceof Buffer) {
if (data.length > 0) {
chunks.push(data);
count = data.length;
}
}
else {
for await (let chunk of data) {
if (chunk.length > 0) {
chunks.push(chunk);
count += chunk.length;
}
}
}
const extentChunk = {
count,
offset: 0,
id: uuid(),
chunks
};
this.logger.info(`MemoryExtentStore:appendExtent() Add chunks to in-memory map. id:${extentChunk.id} count:${count} chunks.length:${chunks.length}`, contextId);
if (!this.chunks.trySet(this.categoryName, extentChunk)) {
throw this.makeError(409, "MemoryExtentStoreAtSizeLimit", `Cannot add an extent chunk to the in-memory store. Size limit of ${this.chunks.sizeLimit()} bytes will be exceeded`, contextId ?? "");
}
this.logger.debug(`MemoryExtentStore:appendExtent() Added chunks to in-memory map. id:${extentChunk.id} `, contextId);
const extent = {
id: extentChunk.id,
locationId: extentChunk.id,
path: extentChunk.id,
size: count,
lastModifiedInMS: Date.now()
};
await this.metadataStore.updateExtent(extent);
this.logger.debug(`MemoryExtentStore:appendExtent() Added new extent to metadata store. id:${extentChunk.id}`, contextId);
return extentChunk;
}
async readExtent(extentChunk, contextId) {
if (extentChunk === undefined || extentChunk.count === 0) {
return new ZeroBytesStream_1.default(0);
}
if (extentChunk.id === IBlobMetadataStore_1.ZERO_EXTENT_ID) {
const subRangeCount = Math.min(extentChunk.count);
return new ZeroBytesStream_1.default(subRangeCount);
}
this.logger.info(`MemoryExtentStore:readExtent() Fetch chunks from in-memory map. id:${extentChunk.id}`, contextId);
const match = this.chunks.get(this.categoryName, extentChunk.id);
if (!match) {
throw new Error(`Extend ${extentChunk.id} does not exist.`);
}
this.logger.debug(`MemoryExtentStore:readExtent() Fetched chunks from in-memory map. id:${match.id} count:${match.count} chunks.length:${match.chunks.length} totalSize:${this.chunks.totalSize()}`, contextId);
const buffer = new stream_1.Readable();
let skip = extentChunk.offset;
let take = extentChunk.count;
let skippedChunks = 0;
let partialChunks = 0;
let readChunks = 0;
for (const chunk of match.chunks) {
if (take === 0) {
break;
}
if (skip > 0) {
if (chunk.length <= skip) {
// this chunk is entirely skipped
skip -= chunk.length;
skippedChunks++;
}
else {
// part of the chunk is included
const end = skip + Math.min(take, chunk.length - skip);
const slice = chunk.slice(skip, end);
buffer.push(chunk.slice(skip, end));
skip = 0;
take -= slice.length;
partialChunks++;
}
}
else {
if (chunk.length > take) {
// all of the chunk is included, up to the count limit
const slice = chunk.slice(0, take);
buffer.push(slice);
take -= slice.length;
partialChunks++;
}
else {
// all of the chunk is included
buffer.push(chunk);
take -= chunk.length;
readChunks++;
}
}
}
buffer.push(null);
this.logger.debug(`MemoryExtentStore:readExtent() Pushed in-memory chunks to Readable stream. id:${match.id} chunks:${readChunks} skipped:${skippedChunks} partial:${partialChunks}`, contextId);
return buffer;
}
async readExtents(extentChunkArray, offset, count, contextId) {
this.logger.info(`MemoryExtentStore:readExtents() Start read from multi extents...`, contextId);
if (count === 0) {
return new ZeroBytesStream_1.default(0);
}
const start = offset; // Start inclusive position in the merged stream
const end = offset + count; // End exclusive position in the merged stream
const streams = [];
let accumulatedOffset = 0; // Current payload offset in the merged stream
for (const chunk of extentChunkArray) {
const nextOffset = accumulatedOffset + chunk.count;
if (nextOffset <= start) {
accumulatedOffset = nextOffset;
continue;
}
else if (end <= accumulatedOffset) {
break;
}
else {
let chunkStart = chunk.offset;
let chunkEnd = chunk.offset + chunk.count;
if (start > accumulatedOffset) {
chunkStart = chunkStart + start - accumulatedOffset; // Inclusive
}
if (end <= nextOffset) {
chunkEnd = chunkEnd - (nextOffset - end); // Exclusive
}
streams.push(await this.readExtent({
id: chunk.id,
offset: chunkStart,
count: chunkEnd - chunkStart
}, contextId));
accumulatedOffset = nextOffset;
}
}
// TODO: What happens when count exceeds merged payload length?
// throw an error of just return as much data as we can?
if (end !== Infinity && accumulatedOffset < end) {
throw new RangeError(
// tslint:disable-next-line:max-line-length
`Not enough payload data error. Total length of payloads is ${accumulatedOffset}, while required data offset is ${offset}, count is ${count}.`);
}
return multistream(streams);
}
async deleteExtents(extents) {
let count = 0;
for (const id of extents) {
this.logger.info(`MemoryExtentStore:deleteExtents() Delete extent:${id}`);
const extent = this.chunks.get(this.categoryName, id);
if (extent) {
this.chunks.delete(this.categoryName, id);
}
await this.metadataStore.deleteExtent(id);
this.logger.debug(`MemoryExtentStore:deleteExtents() Deleted extent:${id} totalSize:${this.chunks.totalSize()}`);
count++;
}
return count;
}
getMetadataStore() {
return this.metadataStore;
}
}
exports.default = MemoryExtentStore;
//# sourceMappingURL=MemoryExtentStore.js.map