@dataql/node
Version:
DataQL core SDK for unified data management with MongoDB and GraphQL - Production Multi-Cloud Ready
246 lines (245 loc) • 9.24 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.MongoDBIntrospector = void 0;
const mongodb_1 = require("mongodb");
const DatabaseIntrospector_js_1 = require("./DatabaseIntrospector.js");
/**
* MongoDB-specific database introspector
*/
class MongoDBIntrospector extends DatabaseIntrospector_js_1.DatabaseIntrospector {
constructor(connection, options = {}) {
super(connection, options);
}
/**
* Introspect the MongoDB database
*/
async introspect() {
try {
console.log(`[DataQL] Starting MongoDB introspection for: ${this.connection.url}`);
await this.connect();
const collections = await this.getCollections();
const filteredCollections = this.filterCollections(collections);
console.log(`[DataQL] Found ${collections.length} collections, analyzing ${filteredCollections.length}`);
const collectionIntrospections = [];
let totalDocuments = 0;
let totalFieldsDiscovered = 0;
let totalIndexes = 0;
for (const collectionName of filteredCollections) {
try {
const introspection = await this.introspectCollection(collectionName);
collectionIntrospections.push(introspection);
totalDocuments += introspection.documentCount;
totalFieldsDiscovered += Object.keys(introspection.fields).length;
totalIndexes += introspection.indexes?.length || 0;
}
catch (error) {
console.warn(`[DataQL] Failed to introspect collection ${collectionName}:`, error);
}
}
// Generate DataQL schemas
const schemas = this.generateDataQLSchemas(collectionIntrospections);
const result = {
databaseName: this.connection.name || this.extractDatabaseName(),
databaseType: "mongodb",
collections: collectionIntrospections,
schemas,
totalDocuments,
introspectionTime: new Date(),
statistics: {
collectionsAnalyzed: collectionIntrospections.length,
documentsAnalyzed: Math.min(totalDocuments, this.options.sampleSize * collectionIntrospections.length),
fieldsDiscovered: totalFieldsDiscovered,
indexesFound: totalIndexes,
},
};
console.log(`[DataQL] Introspection completed. Found ${collectionIntrospections.length} collections with ${totalFieldsDiscovered} fields`);
return { success: true, data: result };
}
catch (error) {
console.error("[DataQL] Introspection failed:", error);
return {
success: false,
error: error.message || "Unknown introspection error",
};
}
finally {
await this.disconnect();
}
}
/**
* Connect to MongoDB
*/
async connect() {
try {
const options = {
maxPoolSize: 10,
serverSelectionTimeoutMS: 5000,
socketTimeoutMS: 45000,
...this.connection.options,
};
this.client = new mongodb_1.MongoClient(this.connection.url, options);
await this.client.connect();
const dbName = this.connection.name || this.extractDatabaseName();
this.db = this.client.db(dbName);
console.log(`[DataQL] Connected to MongoDB database: ${dbName}`);
return this.client;
}
catch (error) {
throw new Error(`Failed to connect to MongoDB: ${error.message}`);
}
}
/**
* Disconnect from MongoDB
*/
async disconnect() {
if (this.client) {
await this.client.close();
this.client = undefined;
this.db = undefined;
console.log("[DataQL] Disconnected from MongoDB");
}
}
/**
* Get list of collections
*/
async getCollections() {
if (!this.db)
throw new Error("Not connected to database");
const collections = await this.db.listCollections().toArray();
return collections
.filter((col) => col.type === "collection") // Exclude views
.map((col) => col.name)
.filter((name) => !name.startsWith("system.")); // Exclude system collections
}
/**
* Introspect a specific collection
*/
async introspectCollection(name) {
if (!this.db)
throw new Error("Not connected to database");
const collection = this.db.collection(name);
// Get document count
const documentCount = await collection.countDocuments();
// Get sample documents
const sampleSize = Math.min(this.options.sampleSize, documentCount);
const pipeline = [{ $sample: { size: sampleSize } }];
const sampleDocs = await collection.aggregate(pipeline).toArray();
console.log(`[DataQL] Analyzing collection '${name}': ${documentCount} documents (sampling ${sampleDocs.length})`);
// Analyze field structure
const fields = this.analyzeDocuments(sampleDocs);
// Get indexes if requested
let indexes;
if (this.options.includeIndexes) {
indexes = await this.getCollectionIndexes(collection);
}
return {
name,
documentCount,
sampleSize: sampleDocs.length,
fields,
indexes,
};
}
/**
* Analyze documents to extract field information
*/
analyzeDocuments(documents) {
const fieldMap = {};
// Collect all field values
for (const doc of documents) {
this.extractFields(doc, fieldMap);
}
// Analyze each field
const fields = {};
for (const [fieldName, values] of Object.entries(fieldMap)) {
fields[fieldName] = this.analyzeFieldType(values, fieldName);
// Determine if field is required (present in most documents)
const presenceRatio = values.length / documents.length;
fields[fieldName].required = presenceRatio > 0.8;
}
return fields;
}
/**
* Extract fields from a document recursively
*/
extractFields(obj, fieldMap, prefix = "") {
if (!obj || typeof obj !== "object")
return;
for (const [key, value] of Object.entries(obj)) {
const fieldName = prefix ? `${prefix}.${key}` : key;
if (!fieldMap[fieldName]) {
fieldMap[fieldName] = [];
}
fieldMap[fieldName].push(value);
// Recursively extract nested fields (up to maxDepth)
if (value &&
typeof value === "object" &&
!Array.isArray(value) &&
prefix.split(".").length < this.options.maxDepth) {
this.extractFields(value, fieldMap, fieldName);
}
}
}
/**
* Get collection indexes
*/
async getCollectionIndexes(collection) {
try {
const indexes = await collection.listIndexes().toArray();
return indexes
.filter((index) => index.name !== "_id_") // Exclude default _id index
.map((index) => ({
name: index.name,
fields: index.key,
unique: index.unique || false,
sparse: index.sparse || false,
partialFilterExpression: index.partialFilterExpression,
}));
}
catch (error) {
console.warn("Failed to get indexes:", error);
return [];
}
}
/**
* Extract database name from connection URL
*/
extractDatabaseName() {
try {
const url = new URL(this.connection.url);
const pathname = url.pathname.replace(/^\//, "");
return pathname || "introspected_db";
}
catch (error) {
return "introspected_db";
}
}
/**
* Override field type detection for MongoDB-specific types
*/
getValueType(value) {
// Handle MongoDB-specific types
if (value && typeof value === "object") {
// MongoDB ObjectId
if (value._bsontype === "ObjectId" ||
(value.constructor && value.constructor.name === "ObjectId")) {
return "ID";
}
// MongoDB Date
if (value instanceof Date) {
return "Timestamp";
}
// MongoDB Decimal128
if (value._bsontype === "Decimal128") {
return "Number";
}
// MongoDB Binary
if (value._bsontype === "Binary") {
return "String"; // Treat as string for now
}
}
// Fall back to base implementation
return super.getValueType(value);
}
}
exports.MongoDBIntrospector = MongoDBIntrospector;