azurite
Version:
An open source Azure Storage API compatible server
368 lines • 16.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const IBlobMetadataStore_1 = require("../persistence/IBlobMetadataStore");
/********** Ranges Merging Strategy ************/
//
// Example:
// Existing ranges: |---| |------| |---------| |-----|
// New range : |----------------------|
//
// 4 existing ranges, and one new request range.
// Above 4 existing ranges, 3 existing ranges are get impacted.
//
// We have 2 merging strategies:
// #1 Merge all impacted ranges
// : |---| |**------------------------****|
// #2 Split first and last impacted existing ranges:
// : |---| |**|----------------------|****|
//
// Question:
// Every range has a pointer to a chunk in one persistency layer extent.
// When implementing Get Page Ranges Diff request, we will assume 2 (sub)ranges
// are same if they pointing to same range of an extent. (Or we can comparing the
// persistency layer payload, but it's not efficiency.)
//
// For #1 strategy, Get Page Ranges Diff will return a larger range scope than
// the actual changed. Above ranges marked as * will point to new allocated extent,
// while in previous snapshot, they still point to old extent.
// One potential workaround is to update these ranges in snapshot blob to pointing
// to same new allocated extent. But this will make implementation of update ranges
// more complex, and also makes snapshot mutable.
// Another concern is that, the * ranges may have large footprint, which is
// time consuming.
//
// For #2 strategy, there will be more and more small ranges. See the |**| and |****|.
// But it's able . We can make the "merging" work to future GC.
//
// We choose #2 strategy, and leave the merging work to future GC, which will merging
// small ranges in background.
//
// TODO for #2 strategy:
// * Resize API: Should remove additional ranges
// * Download Page Blob API after resize (shift): Should taking resize into consideration
// * Clean Ranges API: Same strategy like update ranges
// * Page Blob GC for Ranges Merger: Merge small ranges and extent chunks for page blob and snapshots
// * GC for un-referred extents
class PageBlobRangesManager {
mergeRange(ranges, range) {
const start = range.start; // Inclusive
const end = range.end; // Inclusive
const persistency = range.persistency;
const impactedScope = this.selectImpactedRanges(ranges, start, end);
// Find out first and last impacted range index
const impactedStartIndex = impactedScope[0];
const impactedEndIndex = impactedScope[1];
const impactedRangesCount = impactedStartIndex > impactedEndIndex // No impacted ranges
? 0
: impactedEndIndex - impactedStartIndex + 1;
// New created range for this request payload
const newRange = { start, end, persistency };
if (impactedRangesCount === 0) {
// If there is no existing impacted range, just insert the new range
ranges.splice(impactedStartIndex, 0, newRange);
}
else {
// Otherwise, try to split the first and last impacted ranges
const firstImpactedRange = ranges[impactedStartIndex];
const lastImpactedRange = ranges[impactedEndIndex];
// Ranges to be inserted
const newRanges = [];
// If first range needs to be split, push the split range
if (firstImpactedRange.end >= start && firstImpactedRange.start < start) {
newRanges.push({
start: firstImpactedRange.start,
end: start - 1,
persistency: {
id: firstImpactedRange.persistency.id,
offset: firstImpactedRange.persistency.offset,
count: start - firstImpactedRange.start
}
});
}
newRanges.push(newRange);
// If last impacted range needs to be split, push the split range
if (end >= lastImpactedRange.start && end < lastImpactedRange.end) {
newRanges.push({
start: end + 1,
end: lastImpactedRange.end,
persistency: {
id: lastImpactedRange.persistency.id,
offset: lastImpactedRange.persistency.offset +
(end + 1 - lastImpactedRange.start),
count: lastImpactedRange.end - end
}
});
}
ranges.splice(impactedStartIndex, impactedRangesCount, ...newRanges);
}
}
clearRange(ranges, range) {
const start = range.start; // Inclusive
const end = range.end; // Inclusive
// Find out existing impacted ranges list
const impactedScope = this.selectImpactedRanges(ranges, start, end);
// Find out first and last impacted range index
const impactedStartIndex = impactedScope[0];
const impactedEndIndex = impactedScope[1];
const impactedRangesCount = impactedStartIndex > impactedEndIndex // No impacted ranges
? 0
: impactedEndIndex - impactedStartIndex + 1;
if (impactedRangesCount > 0) {
// Try to split the first and last impacted ranges
const firstImpactedRange = ranges[impactedStartIndex];
const lastImpactedRange = ranges[impactedEndIndex];
// Ranges to be inserted
const newRanges = [];
// If first range needs to be split, push the split range
if (firstImpactedRange.end >= start && firstImpactedRange.start < start) {
newRanges.push({
start: firstImpactedRange.start,
end: start - 1,
persistency: {
id: firstImpactedRange.persistency.id,
offset: firstImpactedRange.persistency.offset,
count: start - firstImpactedRange.start
}
});
}
// If last impacted range needs to be split, push the split range
if (end >= lastImpactedRange.start && end < lastImpactedRange.end) {
newRanges.push({
start: end + 1,
end: lastImpactedRange.end,
persistency: {
id: lastImpactedRange.persistency.id,
offset: lastImpactedRange.persistency.offset +
(end + 1 - lastImpactedRange.start),
count: lastImpactedRange.end - end
}
});
}
ranges.splice(impactedStartIndex, impactedRangesCount, ...newRanges);
}
}
/**
* This method will not modify values of parameter ranges.
*
* @param {PersistencyPageRange[]} ranges
* @param {PageRange} range
* @returns {PersistencyPageRange[]}
* @memberof PageBlobRangesManager
*/
cutRanges(ranges, range) {
const start = range.start; // Inclusive
const end = range.end; // Inclusive
const impactedScope = this.selectImpactedRanges(ranges, start, end);
// Find out first and last impacted range index
const impactedStartIndex = impactedScope[0];
const impactedEndIndex = impactedScope[1];
const impactedRangesCount = impactedStartIndex > impactedEndIndex
? 0 // No impacted ranges
: impactedEndIndex - impactedStartIndex + 1;
let impactedRanges = [];
if (impactedRangesCount > 0) {
impactedRanges = ranges.slice(impactedStartIndex, impactedEndIndex + 1);
// If first range needs to be split
const firstImpactedRange = impactedRanges[0];
if (firstImpactedRange.end >= start && firstImpactedRange.start < start) {
impactedRanges[0] = {
start,
end: firstImpactedRange.end,
persistency: {
offset: start -
firstImpactedRange.start +
firstImpactedRange.persistency.offset,
count: firstImpactedRange.persistency.count -
(start - firstImpactedRange.start),
id: firstImpactedRange.persistency.id
}
};
}
const lastImpactedRange = impactedRanges[impactedRanges.length - 1];
// If last impacted range needs to be split
if (end >= lastImpactedRange.start && end < lastImpactedRange.end) {
impactedRanges[impactedRanges.length - 1] = {
start: lastImpactedRange.start,
end,
persistency: {
offset: lastImpactedRange.persistency.offset,
count: lastImpactedRange.persistency.count -
(lastImpactedRange.end - end),
id: lastImpactedRange.persistency.id
}
};
}
}
return impactedRanges;
}
fillZeroRanges(ranges, range) {
ranges = this.cutRanges(ranges, range);
const filledRanges = [];
if (ranges.length === 0) {
return [
{
start: range.start,
end: range.end,
persistency: {
offset: 0,
count: range.end + 1 - range.start,
id: IBlobMetadataStore_1.ZERO_EXTENT_ID
}
}
];
}
const firstRange = ranges[0];
const lastRange = ranges[ranges.length - 1];
if (range.start < firstRange.start) {
filledRanges.push({
start: range.start,
end: firstRange.start - 1,
persistency: {
offset: 0,
count: firstRange.start - range.start,
id: IBlobMetadataStore_1.ZERO_EXTENT_ID
}
});
}
// TODO: fill in zero ranges in the middle
for (let i = 0; i < ranges.length - 1; i++) {
const nextRange = ranges[i + 1];
const currentRange = ranges[i];
filledRanges.push(currentRange);
const gap = nextRange.start - 1 - currentRange.end;
if (gap > 0) {
filledRanges.push({
start: currentRange.end + 1,
end: nextRange.start - 1,
persistency: {
offset: 0,
count: gap,
id: IBlobMetadataStore_1.ZERO_EXTENT_ID
}
});
}
}
filledRanges.push(ranges[ranges.length - 1]);
if (lastRange.end < range.end) {
filledRanges.push({
start: lastRange.end + 1,
end: range.end,
persistency: {
offset: 0,
count: range.end - lastRange.end,
id: IBlobMetadataStore_1.ZERO_EXTENT_ID
}
});
}
return filledRanges;
}
/**
* Select overlapped ranges based on binary search.
*
* @public
* @param {PersistencyPageRange[]} ranges Existing ranges
* @param {number} start Start offset of the new range, inclusive
* @param {number} end End offset of the new range, inclusive
* @returns {[number, number]} Impacted start and end ranges indexes tuple
* When no impacted ranges found, will return indexes
* for 2 closet existing ranges
* [FirstLargerRangeIndex, FirstSmallerRangeIndex]
* @memberof PageBlobRangesManager
*/
selectImpactedRanges(ranges, start, end) {
if (ranges.length === 0) {
return [Infinity, -1];
}
if (start > end || start < 0) {
throw new RangeError(
// tslint:disable-next-line:max-line-length
"PageBlobRangesManager:selectImpactedRanges() start must less equal than end parameter, start must larger equal than 0.");
}
const impactedRangeStartIndex = this.locateFirstImpactedRange(ranges, 0, ranges.length, start);
const impactedRangesEndIndex = this.locateLastImpactedRange(ranges, 0, ranges.length, end);
return [impactedRangeStartIndex, impactedRangesEndIndex];
}
/**
* Locate first impacted range for a given position.
*
* @public
* @param {PersistencyPageRange[]} ranges
* @param {number} searchStart Index of start range in ranges array, inclusive
* @param {number} searchEnd Index of end range in ranges array, exclusive
* @param {number} position First range index covers or larger than position will be returned
* @returns {number} Index of first impacted range or Infinity for no results
* @memberof PageBlobHandler
*/
locateFirstImpactedRange(ranges, searchStart, searchEnd, position) {
searchStart = searchStart < 0 ? 0 : searchStart;
searchEnd = searchEnd > ranges.length ? searchEnd : searchEnd;
if (ranges.length === 0 || searchStart >= searchEnd) {
return Infinity;
}
// Only last element to check
if (searchStart === searchEnd - 1) {
return this.positionInRange(ranges[searchStart], position) ||
position < ranges[searchStart].start
? searchStart
: Infinity;
}
// 2 or more elements left
const searchMid = Math.floor((searchStart + searchEnd) / 2);
const indexInLeft = this.locateFirstImpactedRange(ranges, searchStart, searchMid, position);
if (indexInLeft !== Infinity) {
return indexInLeft;
}
if (this.positionInRange(ranges[searchMid], position) ||
position < ranges[searchMid].start) {
return searchMid;
}
else {
return this.locateFirstImpactedRange(ranges, searchMid + 1, // Remove searchMid range from searching scope
searchEnd, position);
}
}
/**
* Locate last impacted range for a given position.
*
* @public
* @param {PersistencyPageRange[]} ranges
* @param {number} searchStart Index of start range in ranges array, inclusive
* @param {number} searchEnd Index of end range in ranges array, exclusive
* @param {number} position Last range index covers or less than position will be returned
* @returns {number} Index of first impacted range or -1 for no results
* @memberof PageBlobHandler
*/
locateLastImpactedRange(ranges, searchStart, searchEnd, position) {
searchStart = searchStart < 0 ? 0 : searchStart;
searchEnd = searchEnd > ranges.length ? searchEnd : searchEnd;
if (ranges.length === 0 || searchStart >= searchEnd) {
return -1;
}
// Only last element to check
if (searchStart === searchEnd - 1) {
return this.positionInRange(ranges[searchStart], position) ||
position > ranges[searchStart].end
? searchStart
: -1;
}
// 2 or more elements left
const searchMid = Math.floor((searchStart + searchEnd) / 2);
const indexInRight = this.locateLastImpactedRange(ranges, searchMid + 1, // Remove searchMid range from searching scope
searchEnd, position);
if (indexInRight > -1) {
return indexInRight;
}
if (this.positionInRange(ranges[searchMid], position) ||
position > ranges[searchMid].end) {
return searchMid;
}
else {
return this.locateLastImpactedRange(ranges, searchStart, searchMid, position);
}
}
positionInRange(range, position) {
return position >= range.start && position <= range.end;
}
}
exports.default = PageBlobRangesManager;
//# sourceMappingURL=PageBlobRangesManager.js.map