longcelot-sheet-db
Version:
Google Sheets-backed staging database adapter for Node.js with schema-first design
457 lines • 18.9 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.SheetClient = exports.VALIDATION_CHECK_INTERVAL = exports.VALIDATION_ROW_BUFFER = void 0;
exports.columnIndexToA1Letter = columnIndexToA1Letter;
const googleapis_1 = require("googleapis");
const stream_1 = require("stream");
const DEFAULT_CACHE_TTL_MS = 2000;
/**
* Extra rows past the current data range to pre-apply boolean/enum validation to,
* so a handful of new rows still get checkbox/dropdown UI before the next sync.
* Left unbounded (the GridRange default), validation extends to the sheet's full
* 1000-row default grid, and Sheets API reads then treat every one of those
* formatted-but-empty rows as "has content" — see FAQ.md #10.
*/
exports.VALIDATION_ROW_BUFFER = 200;
/**
* How often CRUDOperations.create() re-checks whether the validated range needs
* extending as rows are appended between syncs (see FAQ.md #10 follow-up). Must be
* at most half of VALIDATION_ROW_BUFFER so coverage never runs out between checks:
* a check at row R extends coverage to R + VALIDATION_ROW_BUFFER; the next check at
* R + VALIDATION_CHECK_INTERVAL must still land inside that window.
*/
exports.VALIDATION_CHECK_INTERVAL = exports.VALIDATION_ROW_BUFFER / 2;
function hexToRgb(hex) {
const normalized = hex.replace('#', '');
const r = parseInt(normalized.substring(0, 2), 16);
const g = parseInt(normalized.substring(2, 4), 16);
const b = parseInt(normalized.substring(4, 6), 16);
return { red: r / 255, green: g / 255, blue: b / 255 };
}
/** Extracts the 1-based row number from an `updatedRange` like `"Sheet1!A12:G12"`. */
function parseRowNumber(updatedRange) {
const match = updatedRange?.match(/![A-Za-z]+(\d+)/);
return match ? parseInt(match[1], 10) : 0;
}
/** Converts a 0-based column index to A1 column letters (0 -> 'A', 25 -> 'Z', 26 -> 'AA'). */
function columnIndexToA1Letter(index) {
let n = index + 1;
let letters = '';
while (n > 0) {
const remainder = (n - 1) % 26;
letters = String.fromCharCode(65 + remainder) + letters;
n = Math.floor((n - 1) / 26);
}
return letters;
}
class SheetClient {
constructor(credentials, tokens, cacheConfig) {
/** getAllRows() results keyed by `${spreadsheetId}::${sheetName}`, valid until expiresAt. */
this._readCache = new Map();
/** Collapses concurrent getAllRows() calls for the same key into a single API request. */
this._inFlightReads = new Map();
this.auth = new googleapis_1.google.auth.OAuth2(credentials.clientId, credentials.clientSecret, credentials.redirectUri);
this.auth.setCredentials(tokens);
this.sheets = googleapis_1.google.sheets({ version: 'v4', auth: this.auth });
this.drive = googleapis_1.google.drive({ version: 'v3', auth: this.auth });
this.cacheEnabled = cacheConfig?.enabled ?? true;
this.cacheTtlMs = cacheConfig?.ttlMs ?? DEFAULT_CACHE_TTL_MS;
}
async createSpreadsheet(title, options) {
const supportsAllDrives = !!(options?.sharedDriveId || options?.folderId);
const parents = options?.folderId
? [options.folderId]
: options?.sharedDriveId
? [options.sharedDriveId]
: undefined;
const response = await this.drive.files.create({
supportsAllDrives,
requestBody: {
name: title,
mimeType: 'application/vnd.google-apps.spreadsheet',
...(parents ? { parents } : {}),
},
fields: 'id',
});
return response.data.id;
}
async findOrCreateFolder(name, parentId, sharedDriveId) {
const escapedName = name.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
let q = `name='${escapedName}' and mimeType='application/vnd.google-apps.folder' and trashed=false`;
if (parentId)
q += ` and '${parentId}' in parents`;
const listParams = {
q,
fields: 'files(id)',
pageSize: 1,
};
if (sharedDriveId) {
listParams.corpora = 'drive';
listParams.driveId = sharedDriveId;
listParams.includeItemsFromAllDrives = true;
listParams.supportsAllDrives = true;
}
const found = await this.drive.files.list(listParams);
if (found.data.files && found.data.files.length > 0) {
return found.data.files[0].id;
}
const createParents = parentId
? [parentId]
: sharedDriveId
? [sharedDriveId]
: undefined;
const created = await this.drive.files.create({
supportsAllDrives: !!sharedDriveId,
requestBody: {
name,
mimeType: 'application/vnd.google-apps.folder',
...(createParents ? { parents: createParents } : {}),
},
fields: 'id',
});
return created.data.id;
}
async uploadFile(buffer, filename, mimeType, folderId, makePublic) {
const readable = new stream_1.Readable();
readable.push(buffer);
readable.push(null);
const response = await this.drive.files.create({
requestBody: {
name: filename,
...(folderId ? { parents: [folderId] } : {}),
},
media: {
mimeType,
body: readable,
},
fields: 'id',
});
const fileId = response.data.id;
if (makePublic) {
await this.drive.permissions.create({
fileId,
requestBody: { type: 'anyone', role: 'reader' },
});
}
return fileId;
}
async deleteFile(fileId) {
await this.drive.files.delete({ fileId });
}
async addSheet(spreadsheetId, sheetName) {
await this.sheets.spreadsheets.batchUpdate({
spreadsheetId,
requestBody: {
requests: [
{
addSheet: {
properties: { title: sheetName },
},
},
],
},
});
}
async getSheetNames(spreadsheetId) {
const response = await this.sheets.spreadsheets.get({ spreadsheetId });
return response.data.sheets?.map((sheet) => sheet.properties?.title || '') || [];
}
async writeHeader(spreadsheetId, sheetName, headers) {
await this.sheets.spreadsheets.values.update({
spreadsheetId,
range: `${sheetName}!A1`,
valueInputOption: 'RAW',
requestBody: {
values: [headers],
},
});
this.invalidateCache(spreadsheetId, sheetName);
}
/**
* Applies header fill color, frozen rows/columns, auto-fit column widths, and
* boolean/enum data validation dropdowns in a single batchUpdate call.
*/
async formatSheet(spreadsheetId, sheetName, options) {
const sheetId = await this.getSheetId(spreadsheetId, sheetName);
const requests = [];
if (options.headerColor) {
requests.push({
repeatCell: {
range: {
sheetId,
startRowIndex: 0,
endRowIndex: 1,
startColumnIndex: 0,
endColumnIndex: options.columnCount,
},
cell: {
userEnteredFormat: { backgroundColor: hexToRgb(options.headerColor) },
},
fields: 'userEnteredFormat.backgroundColor',
},
});
}
if (options.freezeHeader || options.freezeFirstColumn) {
const fields = [];
const gridProperties = {};
if (options.freezeHeader) {
gridProperties.frozenRowCount = 1;
fields.push('gridProperties.frozenRowCount');
}
if (options.freezeFirstColumn) {
gridProperties.frozenColumnCount = 1;
fields.push('gridProperties.frozenColumnCount');
}
requests.push({
updateSheetProperties: {
properties: { sheetId, gridProperties },
fields: fields.join(','),
},
});
}
requests.push({
autoResizeDimensions: {
dimensions: {
sheetId,
dimension: 'COLUMNS',
startIndex: 0,
endIndex: options.columnCount,
},
},
});
const validationEndRowIndex = 1 + (options.dataRowCount ?? 0) + exports.VALIDATION_ROW_BUFFER;
requests.push(...this.buildValidationRequests(sheetId, options.validations ?? [], validationEndRowIndex));
await this.sheets.spreadsheets.batchUpdate({
spreadsheetId,
requestBody: { requests },
});
}
/**
* Re-applies boolean/enum validation rules bounded to `dataRowCount + VALIDATION_ROW_BUFFER`,
* without touching header color/freeze/auto-resize. Called by CRUDOperations.create() as rows
* are appended between syncs, so the validated range keeps pace with real row growth instead
* of only catching up the next time `lsdb sync` runs — see FAQ.md #10 follow-up.
*/
async extendValidation(spreadsheetId, sheetName, validations, dataRowCount) {
if (validations.length === 0)
return;
const sheetId = await this.getSheetId(spreadsheetId, sheetName);
const endRowIndex = 1 + dataRowCount + exports.VALIDATION_ROW_BUFFER;
const requests = this.buildValidationRequests(sheetId, validations, endRowIndex);
await this.sheets.spreadsheets.batchUpdate({
spreadsheetId,
requestBody: { requests },
});
}
buildValidationRequests(sheetId, validations, endRowIndex) {
return validations.map((rule) => ({
setDataValidation: {
range: {
sheetId,
startRowIndex: 1,
endRowIndex,
startColumnIndex: rule.columnIndex,
endColumnIndex: rule.columnIndex + 1,
},
rule: {
condition: {
type: 'ONE_OF_LIST',
values: rule.values.map((v) => ({ userEnteredValue: String(v) })),
},
strict: true,
showCustomUi: true,
},
},
}));
}
/** Returns the 1-based sheet row number the new row was written to (parsed from the API's updatedRange, no extra read). */
async appendRow(spreadsheetId, sheetName, values) {
const response = await this.sheets.spreadsheets.values.append({
spreadsheetId,
range: `${sheetName}!A:A`,
valueInputOption: 'RAW',
requestBody: {
values: [values],
},
});
this.invalidateCache(spreadsheetId, sheetName);
return parseRowNumber(response.data.updates?.updatedRange);
}
async appendRows(spreadsheetId, sheetName, rows) {
if (rows.length === 0)
return;
await this.sheets.spreadsheets.values.append({
spreadsheetId,
range: `${sheetName}!A:A`,
valueInputOption: 'RAW',
requestBody: {
values: rows,
},
});
this.invalidateCache(spreadsheetId, sheetName);
}
/**
* Reads the full `A:ZZ` range for a tab. Every findMany()/findOne()/count()/update()/delete()
* call funnels through here, so a single request handler that touches a table more than once
* (e.g. checkUniqueness() calling findOne() per unique column) — or concurrent requests
* from different users hitting the same catalog table — used to mean one Sheets API read per
* call. That's what exhausts Google's default per-user read quota under any real concurrency.
* A short-TTL cache plus in-flight de-duplication collapses those into a single API call;
* see FAQ.md #11 for the incident and CacheConfig for tuning/disabling it.
*/
async getAllRows(spreadsheetId, sheetName) {
if (!this.cacheEnabled)
return this._fetchAllRows(spreadsheetId, sheetName);
const key = this._cacheKey(spreadsheetId, sheetName);
const cached = this._readCache.get(key);
if (cached && cached.expiresAt > Date.now())
return cached.data;
const inFlight = this._inFlightReads.get(key);
if (inFlight)
return inFlight;
const promise = this._fetchAllRows(spreadsheetId, sheetName)
.then((data) => {
this._readCache.set(key, { data, expiresAt: Date.now() + this.cacheTtlMs });
this._inFlightReads.delete(key);
return data;
})
.catch((err) => {
this._inFlightReads.delete(key);
throw err;
});
this._inFlightReads.set(key, promise);
return promise;
}
async _fetchAllRows(spreadsheetId, sheetName) {
const response = await this.sheets.spreadsheets.values.get({
spreadsheetId,
range: `${sheetName}!A:ZZ`,
});
return response.data.values || [];
}
_cacheKey(spreadsheetId, sheetName) {
return `${spreadsheetId}::${sheetName}`;
}
/** Drops the cached read (if any) for a tab. Called automatically after every write; also exposed for callers that write to a sheet outside this client (e.g. a human editing it directly) and need to force the next read to be fresh. */
invalidateCache(spreadsheetId, sheetName) {
const key = this._cacheKey(spreadsheetId, sheetName);
this._readCache.delete(key);
this._inFlightReads.delete(key);
}
async updateRow(spreadsheetId, sheetName, rowIndex, values) {
await this.sheets.spreadsheets.values.update({
spreadsheetId,
range: `${sheetName}!A${rowIndex}`,
valueInputOption: 'RAW',
requestBody: {
values: [values],
},
});
this.invalidateCache(spreadsheetId, sheetName);
}
async deleteRow(spreadsheetId, sheetName, rowIndex) {
const sheetId = await this.getSheetId(spreadsheetId, sheetName);
await this.sheets.spreadsheets.batchUpdate({
spreadsheetId,
requestBody: {
requests: [
{
deleteDimension: {
range: {
sheetId,
dimension: 'ROWS',
startIndex: rowIndex - 1,
endIndex: rowIndex,
},
},
},
],
},
});
this.invalidateCache(spreadsheetId, sheetName);
}
async shareWithUser(spreadsheetId, email, role = 'writer') {
await this.drive.permissions.create({
fileId: spreadsheetId,
requestBody: {
type: 'user',
role,
emailAddress: email,
},
});
}
/**
* Deletes an entire tab. Unlike the private getSheetId() used elsewhere (which falls back to
* sheetId 0 when a title isn't found — fine for formatting calls, not for a delete), this
* throws if the tab doesn't exist so callers can distinguish "already gone" from "about to
* delete the wrong tab."
*/
async deleteSheet(spreadsheetId, sheetName) {
const sheetId = await this.getSheetIdStrict(spreadsheetId, sheetName);
await this.sheets.spreadsheets.batchUpdate({
spreadsheetId,
requestBody: {
requests: [{ deleteSheet: { sheetId } }],
},
});
this.invalidateCache(spreadsheetId, sheetName);
}
/**
* Deletes one or more columns from a tab in a single batchUpdate. `columnIndexes` may be
* given in any order — they're sorted descending internally so earlier deletions don't shift
* the indexes of later ones within the same batch.
*/
async deleteColumns(spreadsheetId, sheetName, columnIndexes) {
if (columnIndexes.length === 0)
return;
const sheetId = await this.getSheetIdStrict(spreadsheetId, sheetName);
const descending = [...columnIndexes].sort((a, b) => b - a);
const requests = descending.map((columnIndex) => ({
deleteDimension: {
range: {
sheetId,
dimension: 'COLUMNS',
startIndex: columnIndex,
endIndex: columnIndex + 1,
},
},
}));
await this.sheets.spreadsheets.batchUpdate({
spreadsheetId,
requestBody: { requests },
});
this.invalidateCache(spreadsheetId, sheetName);
}
/**
* Overwrites a single header cell in place — used for column rename so existing data rows
* are left untouched (unlike a drop + re-add, which would lose every value in that column).
*/
async updateHeaderCell(spreadsheetId, sheetName, columnIndex, value) {
const column = columnIndexToA1Letter(columnIndex);
await this.sheets.spreadsheets.values.update({
spreadsheetId,
range: `${sheetName}!${column}1`,
valueInputOption: 'RAW',
requestBody: {
values: [[value]],
},
});
this.invalidateCache(spreadsheetId, sheetName);
}
async getSheetId(spreadsheetId, sheetName) {
const response = await this.sheets.spreadsheets.get({ spreadsheetId });
const sheet = response.data.sheets?.find((s) => s.properties?.title === sheetName);
return sheet?.properties?.sheetId || 0;
}
async getSheetIdStrict(spreadsheetId, sheetName) {
const response = await this.sheets.spreadsheets.get({ spreadsheetId });
const sheet = response.data.sheets?.find((s) => s.properties?.title === sheetName);
const sheetId = sheet?.properties?.sheetId;
if (sheetId === undefined || sheetId === null) {
throw new Error(`Sheet tab "${sheetName}" not found in spreadsheet ${spreadsheetId}`);
}
return sheetId;
}
}
exports.SheetClient = SheetClient;
//# sourceMappingURL=sheetClient.js.map