signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
117 lines • 5.48 kB
JavaScript
;
/**
* Stage SQLite buffer rows into a DuckDB TEMP table for federated queries.
*
* The buffer must never be opened by DuckDB's bundled SQLite (ATTACH ... TYPE SQLITE):
* two SQLite libraries in one process cannot see each other's POSIX advisory locks,
* so DuckDB's copy would treat the live database as unused, run WAL recovery, and
* truncate the -shm file under node:sqlite's active mmap — crashing the server with
* SIGBUS on the next large write transaction. Instead, rows are read through the
* buffer's own node:sqlite connection and copied into a per-connection DuckDB temp
* table, which the federated SQL unions with parquet exactly as before.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.stageBufferTable = stageBufferTable;
const sqlite_buffer_1 = require("./sqlite-buffer");
/** Rows copied per read/append cycle — bounds JS heap regardless of window size. */
const STAGING_BATCH_SIZE = 5000;
/** Hard cap on staged rows per query, far above any real buffer window. */
const MAX_STAGED_ROWS = 1000000;
/** Map a declared SQLite column type to the DuckDB column type for staging. */
function duckdbTypeFor(sqliteType) {
const t = sqliteType.toUpperCase();
if (t.includes('INT'))
return 'BIGINT';
if (t.includes('REAL') || t.includes('FLOA') || t.includes('DOUB')) {
return 'DOUBLE';
}
return 'VARCHAR';
}
/**
* Copy the buffer rows a query needs (one path, one context, time-windowed,
* unexported only) into a TEMP table on the given DuckDB connection.
*
* Returns the qualified temp table name to use in FROM clauses, or null when
* the path has no buffer table or no rows match (callers skip the UNION ALL,
* as they did when ATTACH-era subqueries returned null).
*
* The temp table carries all buffer columns, so the existing subquery WHERE
* clauses (context/time/exported/filters) remain valid against it.
*/
async function stageBufferTable(connection, buffer, context, signalkPath, fromIso, toIso, warn) {
const schema = buffer.getTableSchema(signalkPath);
if (!schema || schema.length === 0) {
return null;
}
const tableName = `stage_${(0, sqlite_buffer_1.pathToTableName)(signalkPath)}`;
const qualifiedName = `temp.main.${tableName}`;
const columnTypes = schema.map(col => duckdbTypeFor(col.type));
const columnDefs = schema
.map((col, i) => `"${col.name}" ${columnTypes[i]}`)
.join(', ');
await connection.run(`CREATE OR REPLACE TEMP TABLE ${tableName} (${columnDefs})`);
const appender = await connection.createAppender(tableName, 'main', 'temp');
let totalRows = 0;
let afterId = 0;
try {
for (;;) {
const rows = buffer.getRowsForFederation(signalkPath, context, fromIso, toIso, afterId, STAGING_BATCH_SIZE);
if (rows.length === 0)
break;
for (const row of rows) {
for (let i = 0; i < schema.length; i++) {
const value = row[schema[i].name];
if (value === null || value === undefined) {
appender.appendNull();
}
else if (columnTypes[i] === 'BIGINT') {
// SQLite columns are dynamically typed — tolerate stray non-integer content
const num = typeof value === 'bigint' ? value : Number(value);
if (typeof num === 'bigint') {
appender.appendBigInt(num);
}
else if (Number.isFinite(num)) {
appender.appendBigInt(BigInt(Math.trunc(num)));
}
else {
appender.appendNull();
}
}
else if (columnTypes[i] === 'DOUBLE') {
const num = Number(value);
if (Number.isFinite(num)) {
appender.appendDouble(num);
}
else {
appender.appendNull();
}
}
else {
appender.appendVarchar(String(value));
}
}
appender.endRow();
}
totalRows += rows.length;
afterId = Number(rows[rows.length - 1].id);
if (totalRows >= MAX_STAGED_ROWS) {
// Only fail on genuine overflow — landing exactly on the cap is fine
const overflow = buffer.getRowsForFederation(signalkPath, context, fromIso, toIso, afterId, 1);
if (overflow.length === 0)
break;
warn?.(`[buffer-staging] ${signalkPath}: staged row cap reached (${MAX_STAGED_ROWS}); ` +
`buffer data after id ${afterId} would be omitted from this query`);
throw new Error(`[buffer-staging] ${signalkPath}: buffer rows exceed the staged row cap ` +
`(${MAX_STAGED_ROWS}); refusing to answer with incomplete data`);
}
if (rows.length < STAGING_BATCH_SIZE)
break;
}
appender.flushSync();
}
finally {
appender.closeSync();
}
return totalRows > 0 ? qualifiedName : null;
}
//# sourceMappingURL=buffer-staging.js.map