hikma-engine
Version:
Code Knowledge Graph Indexer - A sophisticated TypeScript-based indexer that transforms Git repositories into multi-dimensional knowledge stores for AI agents
230 lines • 7.78 kB
TypeScript
/**
* @file Responsible for loading processed nodes and edges into the unified SQLite persistence layer.
* Manages data persistence across SQLite database with vector support via sqlite-vec extension.
*/
import { NodeWithEmbedding, Edge } from '../types';
import { ConfigManager } from '../config';
/**
* Loads processed data into the unified SQLite database system.
*/
export declare class DataLoader {
private sqlitePath;
private config;
private logger;
private sqliteClient;
/**
* Initializes the DataLoader with database connection parameters.
* @param {string} sqlitePath - Path to the SQLite database file.
* @param {ConfigManager} config - Configuration manager instance.
*/
constructor(sqlitePath: string, config: ConfigManager);
/**
* Establishes connections to SQLite database.
*/
private connectToDatabases;
/**
* Connects to SQLite with retry logic and circuit breaker.
*/
private connectToSQLite;
/**
* Disconnects from SQLite database.
*/
private disconnectFromDatabases;
/**
* Loads nodes and edges into SQLite enhanced graph storage for deep relationship queries.
* @param {NodeWithEmbedding[]} nodes - Array of nodes.
* @param {Edge[]} edges - Array of edges.
*/
private batchLoadToGraphDB;
/**
* Generate a signature hash for duplicate detection.
*/
private generateSignatureHash;
/**
* Loads node metadata into SQLite for fast lookups and keyword search with transaction management.
* @param {NodeWithEmbedding[]} nodes - Array of nodes.
* @param {Edge[]} edges - Array of edges.
*/
private batchLoadToSqlite;
/**
* Loads nodes with embeddings into unified SQLite storage for both relational and vector operations.
* Uses the existing transaction-based approach with vector storage.
* @param {NodeWithEmbedding[]} nodes - Array of nodes with embeddings.
*/
private batchLoadToSQLiteWithVectors;
/**
* Stores vector embeddings for nodes using the existing SQLiteClient vector operations.
* @param {NodeWithEmbedding[]} nodes - Array of nodes with embeddings.
*/
private storeVectorEmbeddings;
/**
* Groups nodes by their type for efficient processing.
* @param {NodeWithEmbedding[]} nodes - Array of nodes to group.
* @returns {Record<string, NodeWithEmbedding[]>} Nodes grouped by type.
*/
private groupNodesByType;
/**
* Prepares SQLite prepared statements for batch inserts.
* @returns {Record<string, any>} Prepared statements for each table.
*/
private prepareSQLiteStatements;
/**
* Inserts FileNode data into SQLite.
*/
private insertFileNodes;
/**
* Inserts CodeNode data into SQLite.
*/
private insertCodeNodes;
/**
* Inserts CommitNode data into SQLite.
*/
private insertCommitNodes;
/**
* Inserts TestNode data into SQLite.
*/
private insertTestNodes;
/**
* Inserts PullRequestNode data into SQLite.
*/
private insertPullRequestNodes;
/**
* Inserts FunctionNode data into SQLite.
*/
private insertFunctionNodes;
private insertFunctionCalls;
private insertRepositoryNodes;
/**
* Synchronous version for use within transactions.
*/
private insertRepositoryNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertFileNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertCodeNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertCommitNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertTestNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertPullRequestNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertFunctionNodesSync;
/**
* Synchronous version for use within transactions.
*/
private insertFunctionCallsSync;
/**
* Main method to load all nodes and edges into the dual persistence layer.
* @param {NodeWithEmbedding[]} nodes - Array of nodes with embeddings.
* @param {Edge[]} edges - Array of edges.
*/
load(nodes: NodeWithEmbedding[], edges: Edge[]): Promise<{
success: boolean;
results: {
sqlite: {
success: boolean;
error?: string;
};
};
}>;
/**
* Gets statistics about the nodes by type.
* @param {NodeWithEmbedding[]} nodes - Array of nodes.
* @returns {Record<string, number>} Node type statistics.
*/
private getNodeTypeStats;
/**
* Attempts to recover from partial failures by retrying failed operations.
* @param {NodeWithEmbedding[]} nodes - Array of nodes to retry.
* @param {Edge[]} edges - Array of edges to retry.
* @param {string[]} failedDatabases - List of databases that failed.
* @returns {Promise<{success: boolean, results: Record<string, {success: boolean, error?: string}>}>}
*/
retryFailedOperations(nodes: NodeWithEmbedding[], edges: Edge[], failedDatabases: string[]): Promise<{
success: boolean;
results: Record<string, {
success: boolean;
error?: string;
}>;
}>;
/**
* Performs a health check on SQLite database and attempts basic recovery.
* @returns {Promise<{healthy: string[], unhealthy: string[], recovered: string[]}>}
*/
performHealthCheck(): Promise<{
healthy: string[];
unhealthy: string[];
recovered: string[];
}>;
/**
* Verifies SQLite database connectivity before attempting operations.
* @returns {Promise<{sqlite: boolean}>}
*/
verifyDatabaseConnectivity(): Promise<{
sqlite: boolean;
}>;
/**
* Verifies SQLite connection.
*/
private verifySQLiteConnection;
/**
* Implements data consistency checks across databases.
* @param {NodeWithEmbedding[]} nodes - Array of nodes to verify.
* @returns {Promise<{consistent: boolean, issues: string[]}>}
*/
verifyDataConsistency(nodes: NodeWithEmbedding[]): Promise<{
consistent: boolean;
issues: string[];
}>;
/**
* Gets loading statistics and SQLite database health information.
* @returns {Promise<{databases: Record<string, boolean>, lastLoad: Date | null, connectivity: {sqlite: boolean}}>}
*/
getStats(): Promise<{
databases: Record<string, boolean>;
lastLoad: Date | null;
connectivity: {
sqlite: boolean;
};
}>;
/**
* Validates nodes before persistence to ensure data integrity.
* @param nodes - Array of nodes to validate
* @returns Validation result with errors if any
*/
private validateNodes;
/**
* Validates edges before persistence to ensure data integrity.
* @param edges - Array of edges to validate
* @returns Validation result with errors if any
*/
private validateEdges;
/**
* Validates data consistency across nodes and edges.
* @param nodes - Array of nodes
* @param edges - Array of edges
* @returns Validation result with errors if any
*/
private validateDataConsistency;
/**
* Performs comprehensive data validation before persistence.
* @param nodes - Array of nodes to validate
* @param edges - Array of edges to validate
* @returns Validation result with all errors
*/
private performDataValidation;
}
//# sourceMappingURL=data-loader.d.ts.map