signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
203 lines • 8.09 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.DuckDBPool = void 0;
const node_api_1 = require("@duckdb/node-api");
const path = __importStar(require("path"));
const fs = __importStar(require("fs-extra"));
class DuckDBPool {
/**
* Initialize the DuckDB instance and load extensions
* Call this once during plugin startup
*
* @throws Error if initialization fails
*/
static async initialize(homeBaseDir) {
if (this.instance) {
return; // Already initialized
}
// DuckDB defaults its extension/home directory to `$HOME/.duckdb`. On hosts
// where $HOME is read-only — e.g. the Signal K App Store CI sandbox, which
// fails activation with `IO Error: Failed to create directory
// "$HOME/.duckdb": Read-only file system` — the `INSTALL spatial`
// below then aborts. Point DuckDB at a writable dir under the plugin's own
// data directory instead; this also caches downloaded extensions across
// restarts. Falls back to DuckDB's default when no directory is provided.
const config = {};
if (homeBaseDir) {
const duckdbHome = path.join(homeBaseDir, '.duckdb');
await fs.ensureDir(duckdbHome);
config.home_directory = duckdbHome;
config.extension_directory = path.join(duckdbHome, 'extensions');
config.temp_directory = path.join(duckdbHome, 'tmp');
}
// Fully set up on a local variable and only publish to this.instance on
// success, so a failure here (e.g. INSTALL spatial with no network) leaves
// the pool uninitialized and a later initialize() can retry cleanly.
const instance = await node_api_1.DuckDBInstance.create(':memory:', config);
// Load spatial extension once for all future connections
const setupConn = await instance.connect();
try {
// Cap DuckDB memory to prevent OOM when combined with Node's heap
await setupConn.runAndReadAll("SET memory_limit = '512MB';");
await setupConn.runAndReadAll('INSTALL spatial;');
await setupConn.runAndReadAll('LOAD spatial;');
this.instance = instance;
this.initialized = true;
}
finally {
setupConn.disconnectSync();
}
}
/**
* Get a connection from the pool
* The connection shares the same instance, so spatial extension is already loaded
*
* @returns A new connection (closes automatically when no longer referenced)
* @throws Error if pool is not initialized
*/
static async getConnection() {
if (!this.instance) {
throw new Error('DuckDBPool not initialized. Call DuckDBPool.initialize() first.');
}
return await this.instance.connect();
}
/**
* Store the SQLite buffer database path for federated queries.
* Call this after initialize() and after the SQLiteBuffer is created.
*
* The path is used only as a "buffer exists" signal and for diagnostics —
* DuckDB must NEVER open buffer.db itself (see buffer-staging.ts for why
* the old ATTACH approach crashed the server).
*
* @param dbPath Absolute path to the SQLite buffer.db file
*/
static initializeSQLiteBuffer(dbPath) {
this.sqliteDbPath = dbPath;
this.sqliteInitialized = true;
}
/**
* Check if the SQLite buffer path has been configured
*/
static isSQLiteBufferInitialized() {
return this.sqliteInitialized && this.sqliteDbPath !== null;
}
/**
* Get the configured SQLite buffer path (or null)
*/
static getSQLiteBufferPath() {
return this.sqliteDbPath;
}
/**
* Cleanup on plugin shutdown
* Sets the instance to null to allow garbage collection
*/
static async shutdown() {
if (this.instance) {
// DuckDB instances handle cleanup automatically
this.instance = null;
this.initialized = false;
this.s3Initialized = false;
this.sqliteDbPath = null;
this.sqliteInitialized = false;
}
}
/**
* Check if pool is ready
* @returns true if the pool has been initialized
*/
static isInitialized() {
return this.initialized;
}
/**
* Initialize S3 credentials for DuckDB
* This allows DuckDB to query S3 parquet files directly
*
* @param config S3 configuration with credentials and region
* @throws Error if pool is not initialized or S3 setup fails
*/
static async initializeS3(config) {
if (!this.instance) {
throw new Error('DuckDBPool not initialized. Call DuckDBPool.initialize() first.');
}
if (this.s3Initialized) {
return; // Already initialized
}
const connection = await this.instance.connect();
try {
// Install and load httpfs extension for S3 support
await connection.runAndReadAll('INSTALL httpfs;');
await connection.runAndReadAll('LOAD httpfs;');
// Create S3 secret — R2 and self-hosted S3-compatible services (Garage, MinIO)
// typically need ENDPOINT and URL_STYLE 'path'
let endpointClause = config.endpoint
? `,\n ENDPOINT '${config.endpoint.replace(/'/g, "''")}'`
: '';
if (config.urlStyle) {
endpointClause += `,\n URL_STYLE '${config.urlStyle}'`;
}
if (config.endpoint && config.useSSL !== undefined) {
endpointClause += `,\n USE_SSL ${config.useSSL ? 'true' : 'false'}`;
}
const secretSql = `
CREATE OR REPLACE SECRET s3_credentials (
TYPE S3,
KEY_ID '${config.accessKeyId.replace(/'/g, "''")}',
SECRET '${config.secretAccessKey.replace(/'/g, "''")}',
REGION '${config.region.replace(/'/g, "''")}'${endpointClause}
)
`;
await connection.runAndReadAll(secretSql);
this.s3Initialized = true;
}
finally {
connection.disconnectSync();
}
}
/**
* Check if S3 credentials have been initialized
* @returns true if S3 is ready for queries
*/
static isS3Initialized() {
return this.s3Initialized;
}
}
exports.DuckDBPool = DuckDBPool;
DuckDBPool.instance = null;
DuckDBPool.initialized = false;
DuckDBPool.s3Initialized = false;
DuckDBPool.sqliteDbPath = null;
DuckDBPool.sqliteInitialized = false;
//# sourceMappingURL=duckdb-pool.js.map