@apilogic/migration-tool-api/lib/clients/OpensearchClient.js

Universal sql/no-sql database serverless migration tool

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.OpensearchClient = void 0;
const opensearch_1 = require("@opensearch-project/opensearch");
const fs = require("fs");
class OpensearchClient {
    constructor(config) {
        this._client = OpensearchClient.initClient(config);
    }
    getClient() {
        return this._client;
    }
    static initClient(config) {
        const options = this.getOptions(config.config);
        console.log(options);
        return new opensearch_1.Client(options);
    }
    static getOptions(options) {
        const devMode = options.devmode;
        const prefix = options.envPrefix || 'AWS';
        const region = options.region || process.env[`${prefix}_REGION`];
        const host = options.url.host || process.env[`${prefix}_HOST`];
        delete options.region;
        if (!region) {
            throw new TypeError('region is required');
        }
        const config = Object.assign({}, options, {
            node: host,
            auth: {
                username: options.url.username,
                password: options.url.password,
            },
        });
        return config;
    }
    load(payload) {
        this._payload = payload;
        return Promise.resolve(undefined);
    }
    async createIndex(index, workingDir, settingsPath, mappingsPath, alias, modelName) {
        if (!alias) {
            throw new Error(`Alias must be provided!`);
        }
        const indexExist = await this._client.indices.exists({ index });
        console.log('EXISTS ' + indexExist);
        if (!indexExist.body) {
            const settings = JSON.parse(fs.readFileSync(workingDir + settingsPath, 'utf8'));
            console.log('Create index params: ', settings);
            await this._client.indices.create({
                index,
                body: { settings, aliases: { [alias]: {} } },
            });
        }
        const properties = JSON.parse(fs.readFileSync(workingDir + mappingsPath, 'utf-8'));
        let modelId;
        if (this._mappingsNeedModelId(properties)) {
            if (!modelName) {
                throw new Error(`Mappings contain semantic fields without "model_id", but no modelName was provided for index "${index}".`);
            }
            modelId = await this._resolveDeployedModelIdByName(modelName);
        }
        return this._mappingsPayload(index, workingDir, mappingsPath, modelId);
    }
    async upsertIndexAlias(aliasName, newIndexName, oldIndexName) {
        console.log(`Moving alias ${aliasName} from index ${oldIndexName} to index ${newIndexName}`);
        if (oldIndexName) {
            try {
                const requestParams = { index: oldIndexName, name: aliasName };
                console.log(`Delete alias ${aliasName} from index ${oldIndexName}`);
                await this._client.indices.deleteAlias(requestParams);
            }
            catch (error) {
                console.warn(error);
            }
        }
        try {
            const requestParams = { index: newIndexName, name: aliasName };
            console.log(`Adding alias ${aliasName} to index ${newIndexName}`);
            return this._client.indices.putAlias(requestParams);
        }
        catch (error) {
            console.error(error);
        }
    }
    async updateIndex(index, workingDir, mappingsPath, modelName) {
        const properties = JSON.parse(fs.readFileSync(workingDir + mappingsPath, 'utf-8'));
        let modelId;
        if (this._mappingsNeedModelId(properties)) {
            if (!modelName) {
                throw new Error(`Mappings contain semantic fields without "model_id", but no modelName was provided for index "${index}".`);
            }
            modelId = await this._resolveDeployedModelIdByName(modelName);
        }
        return this._mappingsPayload(index, workingDir, mappingsPath, modelId);
    }
    async deleteIndex(index) {
        return this._client.indices.delete({ index });
    }
    async putScript(id, script) {
        if (!id || typeof id !== 'string') {
            throw new Error('putScript: "id" is required');
        }
        if (!script || !script.source || typeof script.source !== 'string') {
            throw new Error('putScript: "source" is required');
        }
        const lang = (script.lang && script.lang.trim()) || 'painless';
        const body = { script: { lang, source: script.source } };
        if (script.params && Object.keys(script.params).length > 0) {
            body.script.params = script.params;
        }
        try {
            return this._client.putScript({ id, body });
        }
        catch (e) {
            const reason = e?.meta?.body?.error?.reason || e?.message || e;
            throw new Error(`Error inserting script "${id}": ${reason}`);
        }
    }
    async deleteScript(id) {
        try {
            return this._client.deleteScript({ id });
        }
        catch (e) {
            throw new Error(`Error deleting script with id: ${id}`);
        }
    }
    async reindexData(sourceIndex, destinationIndex) {
        const destinationIndexResponse = await this._client.indices.exists({ index: destinationIndex });
        const sourceIndexResponse = await this._client.indices.exists({ index: sourceIndex });
        if (destinationIndexResponse.statusCode === 404) {
            throw new Error(`Index does not exist. Cannot reindex data from ${sourceIndex} to non-existent ${destinationIndex}`);
        }
        if (sourceIndexResponse.statusCode === 404) {
            throw new Error(`Source index ${sourceIndex} provided for reindex operation does not exist`);
        }
        try {
            await this._client.reindex({
                requests_per_second: 100,
                body: {
                    source: { index: sourceIndex },
                    dest: { index: destinationIndex },
                },
            });
        }
        catch (e) {
            throw new Error(`Error occured during reindex operation: Source index: ${sourceIndex}, Destination index: ${destinationIndex} with error: ${e}`);
        }
        const sourceIndexAliasesResponse = await this._client.cat.aliases({ v: true, format: 'json' });
        const aliases = sourceIndexAliasesResponse.body
            .filter((row) => row['index'] === sourceIndex)
            .map((row) => ({ name: row['alias'] }));
        try {
            for (const alias of aliases) {
                await this.upsertIndexAlias(alias.name, destinationIndex, sourceIndex);
            }
            return { statusCode: 200, body: aliases };
        }
        catch (e) {
            throw new Error('Error during upserting aliases');
        }
    }
    async migratePretrainedModel(params) {
        const cleanModelName = (params.modelName ?? '').trim();
        const cleanGroupName = ((params.groupName ?? '') || cleanModelName).trim();
        const cleanGroupDesc = (params.groupDescription ?? '').trim();
        if (!cleanModelName)
            throw new Error('migratePretrainedModel: "modelName" is required.');
        if (!cleanGroupName)
            throw new Error('migratePretrainedModel: "groupName" is required (after fallback).');
        const { version, modelFormat, waitForModelIdMs = 120000, pollIntervalMs = 3000, deployTimeoutMs = 180000, deployPollIntervalMs = 3000, } = params;
        const model_group_id = await this._ensureModelGroup(cleanGroupName, cleanGroupDesc);
        const primary = await this._ensureModelByNameAndFormat({
            name: cleanModelName,
            version,
            model_group_id,
            model_format: modelFormat,
            waitForModelIdMs,
            pollIntervalMs,
        });
        let model_id = primary;
        try {
            const state = await this._getModelState(model_id);
            if (!this._isDeployedState(state)) {
                const taskId = await this._deployModel(model_id);
                if (taskId) {
                    await this._waitForDeployment(taskId, deployTimeoutMs, deployPollIntervalMs);
                }
                await this._waitUntilDeployed(model_id, deployTimeoutMs, deployPollIntervalMs);
            }
            return { model_group_id, model_id };
        }
        catch (e) {
            const msg = (e?.message || '') + ' ' + JSON.stringify(e?.reason || e?.meta?.body?.error || '');
            const unsupportedTorch = /Unknown builtin op: aten::/i.test(msg) || /scaled_dot_product_attention/i.test(msg);
            if (modelFormat === 'TORCH_SCRIPT' && unsupportedTorch) {
                const onnxId = await this._ensureModelByNameAndFormat({
                    name: cleanModelName,
                    version,
                    model_group_id,
                    model_format: 'ONNX',
                    waitForModelIdMs,
                    pollIntervalMs,
                });
                const taskId = await this._deployModel(onnxId);
                if (taskId) {
                    await this._waitForDeployment(taskId, deployTimeoutMs, deployPollIntervalMs);
                }
                await this._waitUntilDeployed(onnxId, deployTimeoutMs, deployPollIntervalMs);
                return { model_group_id, model_id: onnxId };
            }
            throw e;
        }
    }
    async _ensureModelByNameAndFormat(args) {
        const { name, version, model_group_id, model_format, waitForModelIdMs, pollIntervalMs } = args;
        let model_id = await this._searchModelIdByName(name, model_format);
        if (!model_id) {
            const reg = await this._registerPretrainedModel({
                name,
                version,
                model_group_id,
                model_format,
            });
            model_id = reg.model_id ?? null;
            if (!model_id && reg.task_id) {
                model_id = await this._waitForModelRegistration(reg.task_id, waitForModelIdMs, pollIntervalMs);
            }
            if (!model_id) {
                const started = Date.now();
                while (Date.now() - started < waitForModelIdMs) {
                    const id = await this._searchModelIdByName(name, model_format);
                    if (id) {
                        model_id = id;
                        break;
                    }
                    await new Promise((r) => setTimeout(r, pollIntervalMs));
                }
                if (!model_id)
                    throw new Error(`Model id not available after register for "${name}" with format ${model_format}`);
            }
        }
        const readyStart = Date.now();
        while (Date.now() - readyStart < 30000) {
            const s = await this._getModelState(model_id);
            if (s)
                break;
            await new Promise((r) => setTimeout(r, 1000));
        }
        const check = await this._getModelState(model_id);
        if (!check)
            throw new Error(`Model ${model_id} not visible yet after registration`);
        return model_id;
    }
    async _waitForModelRegistration(taskId, timeoutMs = 180000, pollMs = 3000) {
        const start = Date.now();
        while (Date.now() - start < timeoutMs) {
            const res = await this._pluginRequest({
                method: 'GET',
                path: `/_plugins/_ml/tasks/${encodeURIComponent(taskId)}`
            });
            const state = res.body?.state || res.body?.task_state || res.body?.model_state;
            const modelId = res.body?.model_id ||
                res.body?.result?.model_id ||
                res.body?.worker_node?.model_id ||
                res.body?.task_details?.model_id;
            if (state === 'COMPLETED' && modelId)
                return modelId;
            if (state === 'FAILED') {
                const reason = res.body?.error?.reason ||
                    res.body?.error ||
                    'Model registration task failed';
                throw new Error(`Model registration failed: ${reason}`);
            }
            await new Promise(r => setTimeout(r, pollMs));
        }
        throw new Error(`Timed out waiting for model registration task ${taskId}`);
    }
    async _waitForDeployment(taskId, timeoutMs = 180000, pollMs = 3000) {
        const start = Date.now();
        while (Date.now() - start < timeoutMs) {
            const res = await this._pluginRequest({
                method: 'GET',
                path: `/_plugins/_ml/tasks/${encodeURIComponent(taskId)}`
            });
            const state = res.body?.state || res.body?.task_state;
            if (state === 'COMPLETED')
                return;
            if (state === 'FAILED') {
                const reason = res.body?.error || res.body;
                throw new Error(`Model deployment failed: ${JSON.stringify(reason)}`);
            }
            await new Promise(r => setTimeout(r, pollMs));
        }
        throw new Error(`Timed out waiting for model deployment task ${taskId}`);
    }
    async _mappingsPayload(index, workingDirectory, mappingsPath, modelId) {
        const properties = JSON.parse(fs.readFileSync(workingDirectory + mappingsPath, 'utf-8'));
        const finalProps = this._injectModelIdIntoSemanticProperties(properties, modelId);
        return this._client.indices.putMapping({ index, body: { properties: finalProps } });
    }
    async _getModelState(modelId) {
        try {
            const res = await this._pluginRequest({
                method: 'GET',
                path: `/_plugins/_ml/models/${encodeURIComponent(modelId)}`,
            });
            return res.body?.model_state ?? res.body?.state ?? null;
        }
        catch (e) {
            const code = e?.meta?.statusCode;
            const reason = e?.meta?.body?.error?.reason || '';
            if (code === 404 || /not\s*find|no such/i.test(reason)) {
                return null;
            }
            throw e;
        }
    }
    _isDeployedState(state) {
        return state === 'DEPLOYED' || state === 'PARTIALLY_DEPLOYED' || state === 'LOADED';
    }
    async _deployModel(modelId) {
        const res = await this._client.transport.request({
            method: 'POST',
            path: `/_plugins/_ml/models/${encodeURIComponent(modelId)}/_deploy`,
        });
        return res?.body?.task_id ?? null;
    }
    async _waitUntilDeployed(modelId, timeoutMs, intervalMs) {
        const started = Date.now();
        while (Date.now() - started < timeoutMs) {
            const state = await this._getModelState(modelId);
            if (this._isDeployedState(state))
                return;
            await new Promise((r) => setTimeout(r, intervalMs));
        }
        const last = await this._getModelState(modelId);
        throw new Error(`Model ${modelId} did not reach a deployed state within ${timeoutMs}ms (last state: ${last ?? 'unknown'})`);
    }
    _injectModelIdIntoSemanticProperties(obj, modelId) {
        if (!obj || typeof obj !== 'object')
            return obj;
        if (obj.type === 'semantic') {
            if (!obj.model_id && modelId)
                obj.model_id = modelId;
            return obj;
        }
        for (const k of Object.keys(obj)) {
            const v = obj[k];
            if (Array.isArray(v)) {
                obj[k] = v.map((item) => this._injectModelIdIntoSemanticProperties(item, modelId));
            }
            else if (typeof v === 'object' && v !== null) {
                obj[k] = this._injectModelIdIntoSemanticProperties(v, modelId);
            }
        }
        return obj;
    }
    _mappingsNeedModelId(obj) {
        if (!obj || typeof obj !== 'object')
            return false;
        if (obj.type === 'semantic' && !obj.model_id)
            return true;
        if (Array.isArray(obj))
            return obj.some((v) => this._mappingsNeedModelId(v));
        for (const k of Object.keys(obj)) {
            if (this._mappingsNeedModelId(obj[k]))
                return true;
        }
        return false;
    }
    async _searchModelIdByName(name, format) {
        const modelName = (name ?? '').trim();
        if (!modelName)
            return null;
        const filters = [{ term: { 'name.keyword': modelName } }];
        if (format) {
            filters.push({ term: { 'model_format.keyword': format } });
        }
        const body = {
            query: {
                bool: {
                    filter: filters,
                    must_not: [{ exists: { field: 'chunk_number' } }],
                },
            },
            size: 1,
            sort: [{ created_time: { order: 'desc' } }],
        };
        try {
            const res = await this._pluginRequest({
                method: 'POST',
                path: '/_plugins/_ml/models/_search',
                body,
            });
            return res.body?.hits?.hits?.[0]?._id ?? null;
        }
        catch (e) {
            const code = e?.meta?.statusCode;
            const type = e?.meta?.body?.error?.type;
            const reason = e?.meta?.body?.error?.reason || '';
            if (code === 404 || type === 'index_not_found_exception' || /no such index/i.test(reason)) {
                return null;
            }
            this._throwWithServerReason('models/_search failed', e, body);
        }
    }
    async _resolveDeployedModelIdByName(modelName, preferred = ['ONNX', 'TORCH_SCRIPT']) {
        for (const fmt of preferred) {
            const id = await this._searchModelIdByName(modelName, fmt);
            if (id) {
                const state = await this._getModelState(id);
                if (state && !this._isDeployedState(state)) {
                    const taskId = await this._deployModel(id);
                    if (taskId)
                        await this._waitForDeployment(taskId, 180000, 3000);
                    await this._waitUntilDeployed(id, 180000, 3000);
                }
                return id;
            }
        }
        throw new Error(`Could not resolve a valid model id for "${modelName}" in formats ${preferred.join(', ')}`);
    }
    async _ensureModelGroup(name, description = '') {
        const groupName = (name ?? '').trim();
        if (!groupName) {
            throw new Error('Model group name is required (got empty/undefined).');
        }
        const body = {
            query: {
                bool: {
                    filter: [{ term: { 'name.keyword': groupName } }],
                },
            },
            size: 1,
            sort: [{ created_time: { order: 'desc' } }],
        };
        try {
            const search = await this._pluginRequest({
                method: 'POST',
                path: '/_plugins/_ml/model_groups/_search',
                body,
            });
            const existing = search.body?.hits?.hits?.[0]?._id ?? null;
            if (existing)
                return existing;
            const create = await this._pluginRequest({
                method: 'POST',
                path: '/_plugins/_ml/model_groups/_register',
                body: { name: groupName, description },
            });
            return create.body.model_group_id;
        }
        catch (e) {
            const code = e?.meta?.statusCode;
            const type = e?.meta?.body?.error?.type;
            const reason = e?.meta?.body?.error?.reason || '';
            if (code === 404 || type === 'index_not_found_exception' || /no such index/i.test(reason)) {
                const create = await this._pluginRequest({
                    method: 'POST',
                    path: '/_plugins/_ml/model_groups/_register',
                    body: { name: groupName, description },
                });
                return create.body.model_group_id;
            }
            this._throwWithServerReason('model_groups/_search failed', e, body);
        }
    }
    async _registerPretrainedModel(args) {
        const res = await this._pluginRequest({
            method: 'POST',
            path: '/_plugins/_ml/models/_register',
            body: args,
        });
        return res.body;
    }
    async _pluginRequest(args) {
        return (await this._client.transport.request(args));
    }
    _throwWithServerReason(prefix, e, body) {
        const serverErr = e?.meta?.body?.error;
        const reason = serverErr?.root_cause?.[0]?.reason ||
            serverErr?.reason ||
            e?.message ||
            String(e);
        const details = body ? ` | body=${JSON.stringify(body)}` : '';
        throw new Error(`${prefix}: ${reason}${details}`);
    }
}
exports.OpensearchClient = OpensearchClient;
//# sourceMappingURL=OpensearchClient.js.map