UNPKG

@comake/skl-js-engine

Version:

Standard Knowledge Language Javascript Engine

1,267 lines (1,163 loc) 72.7 kB
/* eslint-disable no-div-regex */ /* eslint-disable line-comment-position */ /* eslint-disable no-inline-comments */ /* eslint-disable require-unicode-regexp */ /* eslint-disable @typescript-eslint/naming-convention */ import type { OpenApi, OpenApiClientConfiguration, OperationWithPathInfo } from '@comake/openapi-operation-executor'; import { OpenApiOperationExecutor } from '@comake/openapi-operation-executor'; import { getIdFromNodeObjectIfDefined, XSD, type ReferenceNodeObject } from '@comake/rmlmapper-js'; import type { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'; import axios from 'axios'; import type { ContextDefinition, GraphObject, NodeObject } from 'jsonld'; import type { Frame } from 'jsonld/jsonld-spec'; import { JSONPath } from 'jsonpath-plus'; import SHACLValidator from 'rdf-validate-shacl'; import type ValidationReport from 'rdf-validate-shacl/src/validation-report'; import { EngineConstants, OPEN_API_RUNTIME_AUTHORIZATION, PROP_ENTITY_ID, PROP_ENTITY_TYPE, PROP_ENTITY_VALUE, RDF, RDFS, RML_LIST, SHACL } from './constants'; import { globalCustomCapabilities } from './customCapabilities'; import { globalHooks, HookStages, HookTypes } from './hooks/globalHooks'; import type { ExecutionOptions } from './JsExecutor'; import type { ICodeExecutor } from './JsExecutor/types'; import { Logger } from './logger'; import { Mapper } from './mapping/Mapper'; import type { SklEngineOptions } from './SklEngineOptions'; import type { FindOperator } from './storage/FindOperator'; import type { FindAllOptions, FindOneOptions, FindOptionsWhere } from './storage/FindOptionsTypes'; import type { GroupByOptions, GroupByResponse } from './storage/GroupOptionTypes'; import { Exists } from './storage/operator/Exists'; import { In } from './storage/operator/In'; import { InversePath } from './storage/operator/InversePath'; import { Not } from './storage/operator/Not'; import { OneOrMorePath } from './storage/operator/OneOrMorePath'; import { SequencePath } from './storage/operator/SequencePath'; import { ZeroOrMorePath } from './storage/operator/ZeroOrMorePath'; import type { QueryAdapter, RawQueryResult } from './storage/query-adapter/QueryAdapter'; import { SparqlQueryAdapter } from './storage/query-adapter/sparql/SparqlQueryAdapter'; import { PerformanceLogger } from './util/PerformanceLogger'; // Import './util/safeJsonStringify'; import type { Callbacks, Capability, CapabilityConfig, CapabilityMapping, Entity, JSONObject, JSONValue, Mapping, MappingWithInputs, MappingWithInputsReference, MappingWithOutputsMapping, MappingWithParallel, MappingWithSeries, OperationResponse, OrArray, RdfList, SeriesCapabilityArgs, SKLEngineInterface, TriggerMapping } from './util/Types'; import { convertJsonLdToQuads, ensureArray, getValueIfDefined, toJSON } from './util/Util'; import { SKL_DATA_NAMESPACE, SKL_NAMESPACE } from './util/Vocabularies'; export type CapabilityHandler = <T extends OrArray<NodeObject> = OrArray<NodeObject>>( params: JSONObject, capabilityConfig?: CapabilityConfig ) => Promise<T>; export type CapabilityInterface = Record<string, CapabilityHandler>; export type MappingResponseOption<T extends boolean> = T extends true ? JSONObject : NodeObject; export type WriteOptions = { bypassHooks?: boolean }; export class SKLEngine implements SKLEngineInterface { private readonly queryAdapter: QueryAdapter; private readonly functions?: Record<string, (args: any | any[]) => any>; private readonly inputFiles?: Record<string, string>; private readonly globalCallbacks?: Callbacks; private readonly disableValidation?: boolean; public readonly capability: CapabilityInterface; private readonly isDebugMode: boolean; private codeExecutor: ICodeExecutor | undefined; private readonly skdsEndpointUrl: string; private readonly scriptPath: string; public constructor(options: SklEngineOptions) { this.queryAdapter = new SparqlQueryAdapter(options); this.disableValidation = options.disableValidation; this.globalCallbacks = options.callbacks; this.inputFiles = options.inputFiles; this.skdsEndpointUrl = (options as any).endpointUrl; this.scriptPath = (options as any).scriptPath; if (options.functions) { this.functions = Object.fromEntries( Object.entries(options.functions).map(([key, func]) => [ key, (data: Record<string | number, any> = {}) => { // Add the SKL instance to the data object // eslint-disable-next-line @typescript-eslint/no-this-alias data.skl = this; // Call the original function return func(data); } ]) ); } this.isDebugMode = options.debugMode ?? false; Logger.getInstance(this.isDebugMode); // eslint-disable-next-line func-style const getCapabilityHandler = (getTarget: CapabilityInterface, property: string): CapabilityHandler => async <T extends OrArray<NodeObject> = OrArray<NodeObject>>( capabilityArgs: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<T> => this.executeCapabilityByName(property, capabilityArgs, capabilityConfig) as Promise<T>; this.capability = new Proxy({} as CapabilityInterface, { get: getCapabilityHandler }); } public setCodeExecutor(codeExecutor: ICodeExecutor): void { this.codeExecutor = codeExecutor; } public async executeRawQuery<T extends RawQueryResult>(query: string): Promise<T[]> { return await this.queryAdapter.executeRawQuery<T>(query); } public async executeRawUpdate(query: string): Promise<void> { return await this.queryAdapter.executeRawUpdate(query); } public async executeRawConstructQuery(query: string, frame?: Frame): Promise<GraphObject> { return await this.queryAdapter.executeRawConstructQuery(query, frame); } public async find(options?: FindOneOptions): Promise<Entity> { return PerformanceLogger.withSpanRoot('SklEngine.find', async() => { const context = { entities: [], operation: 'find', operationParameters: { options }, sklEngine: this }; await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context); try { const entity = await this.queryAdapter.find(options); if (!entity) { throw new Error(`No schema found with fields matching ${JSON.stringify(options)}`); } const updatedContext = { ...context, entities: [entity]}; const afterHookResult = await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entity); return afterHookResult || entity; } catch (error: unknown) { await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error); throw error; } }, { options }); } public async findBy(where: FindOptionsWhere, notFoundErrorMessage?: string): Promise<Entity> { return PerformanceLogger.withSpanRoot('SklEngine.findBy', async() => { const context = { entities: [], operation: 'findBy', operationParameters: { where }, sklEngine: this }; await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context); try { const entity = await this.queryAdapter.findBy(where); if (entity) { const updatedContext = { ...context, entities: [entity]}; await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entity); return entity; } throw new Error(notFoundErrorMessage ?? `No schema found with fields matching ${JSON.stringify(where)}`); } catch (error: unknown) { await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error); throw error; } }, { where }); } public async findByIfExists(options: FindOptionsWhere): Promise<Entity | undefined> { try { const entity = await this.findBy(options); return entity; } catch { return undefined; } } public async findAll(options?: FindAllOptions): Promise<Entity[]> { return PerformanceLogger.withSpanRoot('SklEngine.findAll', async() => { const context = { entities: [], operation: 'findAll', operationParameters: { options }, sklEngine: this }; await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context); try { const entities = await this.queryAdapter.findAll(options); const updatedContext = { ...context, entities }; await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entities); return entities; } catch (error: unknown) { await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error); throw error; } }, { options }); } public async groupBy(options: GroupByOptions): Promise<GroupByResponse> { return PerformanceLogger.withSpanRoot('SklEngine.groupBy', async() => { const context = { entities: [], operation: 'groupBy', operationParameters: { options }, sklEngine: this }; await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context); try { const result = await this.queryAdapter.groupBy(options); const updatedContext = { ...context, result }; await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, result); return result; } catch (error: unknown) { await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error); throw error; } }, { options }); } public async findAllBy(where: FindOptionsWhere): Promise<Entity[]> { return PerformanceLogger.withSpanRoot('SklEngine.findAllBy', async() => { const context = { entities: [], operation: 'findAllBy', operationParameters: { where }, sklEngine: this }; await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context); try { const entities = await this.queryAdapter.findAllBy(where); const updatedContext = { ...context, entities }; await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entities); return entities; } catch (error: unknown) { await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error); throw error; } }, { where }); } public async exists(options?: FindAllOptions): Promise<boolean> { return PerformanceLogger.withSpanRoot('SklEngine.exists', async() => this.queryAdapter.exists(options), { options }); } public async count(options?: FindAllOptions): Promise<number> { return PerformanceLogger.withSpanRoot('SklEngine.count', async() => this.queryAdapter.count(options), { options }); } public async save(entity: Entity, options?: WriteOptions): Promise<Entity>; public async save(entities: Entity[], options?: WriteOptions): Promise<Entity[]>; public async save(entityOrEntities: Entity | Entity[], options?: WriteOptions): Promise<Entity | Entity[]> { return PerformanceLogger.withSpanRoot('SklEngine.save', async() => { const entityArray = Array.isArray(entityOrEntities) ? entityOrEntities : [entityOrEntities]; const isSingleEntity = !Array.isArray(entityOrEntities); await globalHooks.executeBeforeCreate(entityArray, { sklEngine: this, bypassHooks: options?.bypassHooks }); try { await this.validateEntitiesConformToObjectSchema(entityArray); const savedEntities = await this.queryAdapter.save(entityArray); await globalHooks.executeAfterCreate(savedEntities, { sklEngine: this, bypassHooks: options?.bypassHooks }); return isSingleEntity ? savedEntities[0] : savedEntities; } catch (error) { await globalHooks.executeErrorCreate(entityArray, error as Error, { sklEngine: this, bypassHooks: options?.bypassHooks }); throw error; } }, { entityCount: Array.isArray(entityOrEntities) ? entityOrEntities.length : 1 }); } public async update(id: string, attributes: Partial<Entity>, options?: WriteOptions): Promise<void>; public async update(ids: string[], attributes: Partial<Entity>, options?: WriteOptions): Promise<void>; public async update(idOrIds: string | string[], attributes: Partial<Entity>, options?: WriteOptions): Promise<void> { return PerformanceLogger.withSpanRoot('SklEngine.update', async() => { const idArray = Array.isArray(idOrIds) ? idOrIds : [idOrIds]; const isSingleEntity = !Array.isArray(idOrIds); await globalHooks.execute(HookTypes.UPDATE, HookStages.BEFORE, { entities: [], operation: 'update', operationParameters: { idArray, attributes }, sklEngine: this, bypassHooks: options?.bypassHooks }); try { if (idArray.length > 1) { await this.validateEntitiesWithIdsConformsToObjectSchemaForAttributes(idArray, attributes); } else { await this.validateEntityWithIdConformsToObjectSchemaForAttributes(idArray[0], attributes); } await this.queryAdapter.update(isSingleEntity ? idArray[0] : (idArray as any), attributes); await globalHooks.execute(HookTypes.UPDATE, HookStages.AFTER, { entities: [], operation: 'update', operationParameters: { idArray, attributes }, sklEngine: this, bypassHooks: options?.bypassHooks }); } catch (error: unknown) { await globalHooks.execute(HookTypes.UPDATE, HookStages.ERROR, { entities: [], operation: 'update', operationParameters: { idArray, attributes }, sklEngine: this, bypassHooks: options?.bypassHooks }, error); throw error; } }, { idCount: Array.isArray(idOrIds) ? idOrIds.length : 1 }); } public async validateEntitiesConformToObjectSchema(entities: Entity[]): Promise<void> { const entitiesByType = this.groupEntitiesByType(entities); for (const type of Object.keys(entitiesByType)) { const object = await this.findByIfExists({ id: type }); if (object) { const parentObjects = await this.getSuperClassesOfObject(type); for (const currentObject of [object, ...parentObjects]) { const entitiesOfType = entitiesByType[type]; const nounSchemaWithTarget = { ...currentObject, [SHACL.targetNode]: entitiesOfType.map( (entity): ReferenceNodeObject => ({ [PROP_ENTITY_ID]: entity[PROP_ENTITY_ID] }) ) }; const report = await this.convertToQuadsAndValidateAgainstShape(entitiesOfType, nounSchemaWithTarget); if (!report.conforms) { const entityIds = entitiesOfType.map((entity): string => entity[PROP_ENTITY_ID]); this.throwValidationReportError( report, `Entity ${entityIds.join(', ')} does not conform to the ${currentObject[PROP_ENTITY_ID]} schema.` ); } } } } } private groupEntitiesByType(entities: Entity[]): Record<string, Entity[]> { return entities.reduce((groupedEntities: Record<string, Entity[]>, entity): Record<string, Entity[]> => { const entityTypes = Array.isArray(entity[PROP_ENTITY_TYPE]) ? entity[PROP_ENTITY_TYPE] : [entity[PROP_ENTITY_TYPE]]; for (const type of entityTypes) { if (!groupedEntities[type]) { groupedEntities[type] = []; } groupedEntities[type].push(entity); } return groupedEntities; }, {}); } private async getSuperClassesOfObject(object: string): Promise<Entity[]> { return await this.getParentsOfSelector(object); } private async getSuperClassesOfObjects(nouns: string[]): Promise<Entity[]> { return await this.getParentsOfSelector(In(nouns)); } private async getParentsOfSelector(selector: string | FindOperator<any, any>): Promise<Entity[]> { return await this.findAll({ where: { id: InversePath({ subPath: OneOrMorePath({ subPath: RDFS.subClassOf as string }), value: selector }) } }); } private async validateEntityConformsToObjectSchema(entity: Entity): Promise<void> { const nounIds = Array.isArray(entity[PROP_ENTITY_TYPE]) ? entity[PROP_ENTITY_TYPE] : [entity[PROP_ENTITY_TYPE]]; const directObjects = await this.findAllBy({ id: In(nounIds) }); if (directObjects.length > 0) { const existingObjectIds = directObjects.map((object): string => object[PROP_ENTITY_ID]); const parentObjects = await this.getSuperClassesOfObjects(existingObjectIds); for (const currentObject of [...directObjects, ...parentObjects]) { const nounSchemaWithTarget = { ...currentObject, [SHACL.targetNode]: { [PROP_ENTITY_ID]: entity[PROP_ENTITY_ID] } }; const report = await this.convertToQuadsAndValidateAgainstShape(entity, nounSchemaWithTarget); if (!report.conforms) { this.throwValidationReportError( report, `Entity ${entity[PROP_ENTITY_ID]} does not conform to the ${currentObject[PROP_ENTITY_ID]} schema.` ); } } } } private async validateEntitiesWithIdsConformsToObjectSchemaForAttributes( ids: string[], attributes: Partial<Entity> ): Promise<void> { for (const id of ids) { await this.validateEntityWithIdConformsToObjectSchemaForAttributes(id, attributes); } } private async getObjectsAndParentObjectsOfEntity(id: string): Promise<Entity[]> { return await this.findAllBy({ id: InversePath({ subPath: SequencePath({ subPath: [RDF.type, ZeroOrMorePath({ subPath: RDFS.subClassOf as string })] }), value: id }) }); } private async validateEntityWithIdConformsToObjectSchemaForAttributes( id: string, attributes: Partial<Entity> ): Promise<void> { const nouns = await this.getObjectsAndParentObjectsOfEntity(id); for (const currentObject of nouns) { if (SHACL.property in currentObject) { const nounProperties = ensureArray(currentObject[SHACL.property] as OrArray<NodeObject>).filter( (property): boolean => { const path = property[SHACL.path]; if (typeof path === 'string' && path in attributes) { return true; } if (typeof path === 'object' && PROP_ENTITY_ID in path! && (path[PROP_ENTITY_ID] as string) in attributes) { return true; } return false; } ); if (nounProperties.length > 0) { const nounSchemaWithTarget = { [PROP_ENTITY_TYPE]: SHACL.NodeShape, [SHACL.targetNode]: { [PROP_ENTITY_ID]: id }, [SHACL.property]: nounProperties }; const attributesWithId = { ...attributes, [PROP_ENTITY_ID]: id }; const report = await this.convertToQuadsAndValidateAgainstShape(attributesWithId, nounSchemaWithTarget); if (!report.conforms) { this.throwValidationReportError( report, `Entity ${id} does not conform to the ${currentObject[PROP_ENTITY_ID]} schema.` ); } } } } } public async delete(id: string, options?: WriteOptions): Promise<void>; public async delete(ids: string[], options?: WriteOptions): Promise<void>; public async delete(idOrIds: string | string[], options?: WriteOptions): Promise<void> { return PerformanceLogger.withSpanRoot('SklEngine.delete', async() => { const idArray = Array.isArray(idOrIds) ? idOrIds : [idOrIds]; await globalHooks.execute(HookTypes.DELETE, HookStages.BEFORE, { entities: [], operation: 'delete', operationParameters: { idArray }, sklEngine: this, bypassHooks: options?.bypassHooks }); try { await this.queryAdapter.delete(idArray); await globalHooks.execute(HookTypes.DELETE, HookStages.AFTER, { entities: [], operation: 'delete', operationParameters: { idArray }, sklEngine: this, bypassHooks: options?.bypassHooks }); } catch (error) { await globalHooks.execute(HookTypes.DELETE, HookStages.ERROR, { entities: [], operation: 'delete', operationParameters: { idArray }, sklEngine: this, bypassHooks: options?.bypassHooks }, error); throw error; } }, { idCount: Array.isArray(idOrIds) ? idOrIds.length : 1 }); } public async destroy(entity: Entity): Promise<Entity>; public async destroy(entities: Entity[]): Promise<Entity[]>; public async destroy(entityOrEntities: Entity | Entity[]): Promise<Entity | Entity[]> { if (Array.isArray(entityOrEntities)) { return await this.queryAdapter.destroy(entityOrEntities); } return await this.queryAdapter.destroy(entityOrEntities); } public async destroyAll(): Promise<void> { return await this.queryAdapter.destroyAll(); } public async performMapping( args: JSONValue, mapping: OrArray<NodeObject>, frame?: Record<string, any>, capabilityConfig?: CapabilityConfig, jsExecutionOptions?: ExecutionOptions ): Promise<NodeObject> { const mappingArray = ensureArray(mapping); const codeBlocks = mappingArray.filter( (mappingItem: NodeObject): boolean => mappingItem[PROP_ENTITY_TYPE] === EngineConstants.spec.codeBlock && this.isJavaScriptCode(getValueIfDefined(mappingItem[EngineConstants.prop.codeBody])!) ); // FIXME: Handle if we can combine the codeb blocks with triples map. // As of now if there is any code block, triples map does not get executed. if (codeBlocks.length > 0) { return await this.executeCodeBlocks(codeBlocks, args, jsExecutionOptions ?? {}); } const functions = { ...this.functions, ...capabilityConfig?.functions }; const mapper = new Mapper({ functions }); return await mapper.apply(args, mapping, frame ?? {}); } public async executeTrigger(integration: string, payload: any): Promise<void> { const triggerToCapabilityMapping = await this.findTriggerCapabilityMapping(integration); const capabilityArgs = await this.performParameterMappingOnArgsIfDefined( payload, triggerToCapabilityMapping as Partial<MappingWithInputs> | Partial<MappingWithInputsReference> ); const capabilityId = await this.performCapabilityMappingWithArgs(payload, triggerToCapabilityMapping); if (capabilityId) { const mappedCapability = (await this.findBy({ id: capabilityId })) as Capability; await this.executeCapability(mappedCapability, capabilityArgs); } } private async findTriggerCapabilityMapping(integration: string): Promise<TriggerMapping> { const triggerCapabilityMappingNew = (await this.findBy( { type: EngineConstants.spec.capabilityMapping, [EngineConstants.prop.capability]: integration, [EngineConstants.prop.capabilityType]: EngineConstants.spec.triggerCapabilityMapping }, `Failed to find a Trigger Capability mapping for integration ${integration}` )) as TriggerMapping; if (triggerCapabilityMappingNew) { return triggerCapabilityMappingNew; } throw new Error(`Failed to find a Trigger Capability mapping for integration ${integration}`); } private async executeCapabilityByName( capabilityName: string, capabilityArgs: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<OrArray<NodeObject>> { const capability = await this.findCapabilityWithName(capabilityName); return await this.executeCapability(capability, capabilityArgs, capabilityConfig); } private async findCapabilityWithName(capabilityName: string): Promise<Capability> { return (await this.findBy( { type: EngineConstants.spec.capability, [EngineConstants.prop.label]: capabilityName }, `Failed to find the capability ${capabilityName} in the schema.` )) as Capability; } public async executeCapability( capability: Capability, capabilityArgs: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<OrArray<NodeObject>> { this.globalCallbacks?.onCapabilityStart?.(capability[PROP_ENTITY_ID], capabilityArgs); if (capabilityConfig?.callbacks?.onCapabilityStart) { Logger.getInstance().log('Capability arguments', capabilityArgs); capabilityConfig.callbacks.onCapabilityStart(capability[PROP_ENTITY_ID], capabilityArgs); } const { mapping, account } = await this.findMappingForCapabilityContextually( capability[PROP_ENTITY_ID], capabilityArgs ); Logger.getInstance().log('Mapping', JSON.stringify(mapping)); const shouldValidate = this.shouldValidate(capabilityConfig); if (shouldValidate) { await this.assertCapabilityParamsMatchParameterSchemas(capabilityArgs, capability); } try { // Execute capability mapping before hook if appropriate - // works for any mapping that can be used as a verb mapping if (mapping) { await globalHooks.executeBeforeExecuteCapabilityMapping([capabilityArgs] as Entity[], mapping, { sklEngine: this }); } const verbReturnValue = await this.executeMapping(mapping, capabilityArgs, capabilityConfig, account); if (shouldValidate) { await this.assertCapabilityReturnValueMatchesReturnTypeSchema(verbReturnValue, capability); } // Execute capability mapping after hook if appropriate if (mapping) { await globalHooks.executeAfterExecuteCapabilityMapping([capabilityArgs] as Entity[], mapping, verbReturnValue, { sklEngine: this }); } this.globalCallbacks?.onCapabilityEnd?.(capability[PROP_ENTITY_ID], verbReturnValue); if (capabilityConfig?.callbacks?.onCapabilityEnd) { capabilityConfig.callbacks.onCapabilityEnd(capability[PROP_ENTITY_ID], verbReturnValue); } return verbReturnValue; } catch (error) { // Execute capability mapping error hook if appropriate if (mapping) { await globalHooks.executeErrorExecuteCapabilityMapping([capabilityArgs] as Entity[], mapping, error as Error, { sklEngine: this }); } throw error; } } private async findMappingForCapabilityContextually( capabilityId: string, args: JSONObject ): Promise<{ mapping: CapabilityMapping; account?: Entity }> { if (args.mapping) { const mapping = await this.findByIfExists({ id: args.mapping as string }); if (!mapping) { throw new Error(`Mapping ${args.mapping as string} not found.`); } return { mapping: mapping as CapabilityMapping }; } if (args.object) { const mapping = await this.findCapabilityObjectMapping(capabilityId, args.object as string); if (mapping) { return { mapping }; } } if (args.account) { const account = await this.findBy({ id: args.account as string }); const integratedProductId = (account[EngineConstants.prop.integration] as ReferenceNodeObject)[PROP_ENTITY_ID]; const mapping = await this.findCapabilityIntegrationMapping(capabilityId, integratedProductId); if (mapping) { return { mapping, account }; } } const mappings = await this.findAllBy({ type: EngineConstants.spec.capabilityMapping, [EngineConstants.prop.capability]: capabilityId, [EngineConstants.prop.integration]: Not(Exists()), [EngineConstants.prop.object]: Not(Exists()) }); if (mappings.length === 1) { return { mapping: mappings[0] as CapabilityMapping }; } if (mappings.length > 1) { throw new Error('Multiple mappings found for capability, please specify one.'); } if (args.object) { throw new Error(`Mapping between object ${args.object as string} and capability ${capabilityId} not found.`); } if (args.account) { throw new Error(`Mapping between account ${args.account as string} and capability ${capabilityId} not found.`); } throw new Error(`No mapping found.`); } public async executeMapping( mapping: Mapping, args: JSONObject, capabilityConfig?: CapabilityConfig, account?: Entity ): Promise<OrArray<NodeObject>> { args = await this.addPreProcessingMappingToArgs(mapping, args, capabilityConfig); let returnValue: OrArray<NodeObject>; // If (EngineConstants.prop.capability in mapping || EngineConstants.prop.capabilityMapping in mapping) { // const capabilityId = await this.performCapabilityMappingWithArgs(args, mapping, capabilityConfig); // const mappedArgs = await this.performParameterMappingOnArgsIfDefined( // { ...args, capabilityId }, // mapping as MappingWithInputs, // capabilityConfig // ); // Logger.getInstance().log('Mapped args', mappedArgs); // returnValue = await this.executeCapabilityMapping(mapping, args, mappedArgs, capabilityConfig); // } else { const mappedArgs = await this.performParameterMappingOnArgsIfDefined( args, mapping as MappingWithInputs, capabilityConfig ); Logger.getInstance().log('Mapped args', mappedArgs); if (EngineConstants.prop.operationId in mapping || EngineConstants.prop.operationMapping in mapping) { returnValue = (await this.executeOperationMapping( mapping, mappedArgs, args, account!, capabilityConfig )) as NodeObject; } else if (EngineConstants.prop.series in mapping) { returnValue = await this.executeSeriesMapping(mapping as MappingWithSeries, mappedArgs, capabilityConfig); } else if (EngineConstants.prop.parallel in mapping) { returnValue = await this.executeParallelMapping(mapping as MappingWithParallel, mappedArgs, capabilityConfig); } else { returnValue = mappedArgs; } // } return await this.performReturnValueMappingWithFrameIfDefined( returnValue as JSONValue, mapping as MappingWithOutputsMapping, capabilityConfig ); } private shouldValidate(capabilityConfig?: CapabilityConfig): boolean { return capabilityConfig?.disableValidation === undefined ? this.disableValidation !== true : !capabilityConfig.disableValidation; } private async executeOperationMapping( mapping: Mapping, mappedArgs: JSONObject, originalArgs: JSONObject, account: Entity, capabilityConfig?: CapabilityConfig ): Promise<OperationResponse | OrArray<NodeObject>> { const integration = (mapping as CapabilityMapping)[EngineConstants.prop.integration]?.[PROP_ENTITY_ID]; // If the mapping has an integration, it means that the operation is an integration operation if (integration) { const operationInfo = await this.performOperationMappingWithArgs(originalArgs, mapping, capabilityConfig); const response = await this.performOperation(operationInfo, mappedArgs, originalArgs, account, capabilityConfig); if (!this.ifCapabilityStreaming(capabilityConfig)) { Logger.getInstance().log('Original response', JSON.stringify(response)); } return response; } // If the mapping does not have an integration, it means that the operation is a capability operation return await this.executeCapabilityMapping(mapping, originalArgs, mappedArgs, capabilityConfig); } private async executeSeriesMapping( mapping: MappingWithSeries, args: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<OrArray<NodeObject>> { const seriesCapabilityMappingsList = this.rdfListToArray(mapping[EngineConstants.prop.series]!); const seriesCapabilityArgs = { originalCapabilityParameters: args, previousCapabilityReturnValue: {}, allStepsResults: [] }; return await this.executeSeriesFromList(seriesCapabilityMappingsList, seriesCapabilityArgs, capabilityConfig); } private rdfListToArray(list: { [RML_LIST]: CapabilityMapping[] } | RdfList<CapabilityMapping>): CapabilityMapping[] { if (!(RML_LIST in list)) { return [ list[RDF.first], ...getIdFromNodeObjectIfDefined(list[RDF.rest] as ReferenceNodeObject) === RDF.nil ? [] : this.rdfListToArray(list[RDF.rest] as RdfList<CapabilityMapping>) ]; } return list[RML_LIST]; } private async executeSeriesFromList( list: Mapping[], args: SeriesCapabilityArgs, capabilityConfig?: CapabilityConfig ): Promise<OrArray<NodeObject>> { const nextCapabilityMapping = list[0]; const returnValue = await this.executeMapping(nextCapabilityMapping, args, capabilityConfig); if (list.length > 1) { return await this.executeSeriesFromList( list.slice(1), { ...args, previousCapabilityReturnValue: returnValue as JSONObject, allStepsResults: [...args.allStepsResults ?? [], returnValue as JSONObject] }, capabilityConfig ); } return returnValue; } private async executeCapabilityMapping( capabilityMapping: Mapping, originalArgs: JSONObject, mappedArgs: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<OrArray<NodeObject>> { const capabilityId = await this.performCapabilityMappingWithArgs(originalArgs, capabilityMapping, capabilityConfig); if (capabilityId) { if (capabilityId === EngineConstants.dataSource.update) { await this.updateEntityFromcapabilityArgs(mappedArgs); return {}; } if (capabilityId === EngineConstants.dataSource.save) { return await this.saveEntityOrEntitiesFromcapabilityArgs(mappedArgs); } if (capabilityId === EngineConstants.dataSource.destroy) { return await this.destroyEntityOrEntitiesFromcapabilityArgs(mappedArgs); } if (capabilityId === EngineConstants.dataSource.findAll) { return await this.findAll(mappedArgs); } if (capabilityId === EngineConstants.dataSource.find) { return await this.find(mappedArgs); } if (capabilityId === EngineConstants.dataSource.count) { return await this.countAndWrapValueFromcapabilityArgs(mappedArgs); } if (capabilityId === EngineConstants.dataSource.exists) { return await this.existsAndWrapValueFromcapabilityArgs(mappedArgs); } if (capabilityId === 'https://skl.ai/capability/execute-code') { const codeBlocks = ensureArray((capabilityMapping[EngineConstants.prop.codeBlocks] as any[]) ?? []).filter( (mappingItem: any): boolean => mappingItem[PROP_ENTITY_TYPE] === EngineConstants.spec.codeBlock && this.isJavaScriptCode(getValueIfDefined(mappingItem[EngineConstants.prop.codeBody])!) ); return await this.executeCodeBlocks(codeBlocks, mappedArgs, {}); } // Check for custom capabilities if (globalCustomCapabilities.has(capabilityId)) { return await globalCustomCapabilities.execute(capabilityId, mappedArgs, this, capabilityConfig); } return await this.findAndExecuteCapability(capabilityId, mappedArgs, capabilityConfig); } return {}; } private async addPreProcessingMappingToArgs( capabilityMapping: Mapping, args: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<JSONObject> { if (EngineConstants.prop.preProcessingMapping in capabilityMapping) { const preMappingArgs = await this.performMapping( args, capabilityMapping[EngineConstants.prop.preProcessingMapping] as NodeObject, getValueIfDefined(capabilityMapping[EngineConstants.prop.preProcessingMappingFrame]), capabilityConfig ); return { ...args, preProcessedParameters: preMappingArgs as JSONObject }; } return args; } private replaceTypeAndId(entity: Record<string, any>): Record<string, any> { if (typeof entity !== 'object') { throw new Error('Entity is not an object'); } const clonedEntity = structuredClone(entity); if (clonedEntity[EngineConstants.prop.type]) { clonedEntity[PROP_ENTITY_TYPE] = clonedEntity[EngineConstants.prop.type]; } if (clonedEntity[EngineConstants.prop.identifier]) { clonedEntity[PROP_ENTITY_ID] = SKL_DATA_NAMESPACE + (clonedEntity[EngineConstants.prop.identifier] as string); } return clonedEntity; } private async updateEntityFromcapabilityArgs(args: Record<string, any>): Promise<void> { let ids = args.id ?? args.ids; if (!Array.isArray(ids)) { ids = [ids]; } // FIX: Temporary fix for the issue where the id always getting prefixed with the namespace ids = ids.map((id: string) => (id.startsWith('http') ? id : `${SKL_DATA_NAMESPACE}${id}`)); await this.update(ids, args.attributes); } private async saveEntityOrEntitiesFromcapabilityArgs(args: Record<string, any>): Promise<OrArray<Entity>> { if (args.entity && typeof args.entity === 'object') { args.entity = this.replaceTypeAndId(args.entity); } if (args.entities && Array.isArray(args.entities)) { args.entities = args.entities.map(this.replaceTypeAndId); } return await this.save(args.entity ?? args.entities); } private async destroyEntityOrEntitiesFromcapabilityArgs(args: Record<string, any>): Promise<OrArray<Entity>> { if (args.entity && typeof args.entity === 'object') { args.entity = this.replaceTypeAndId(args.entity); } if (args.entities && Array.isArray(args.entities)) { args.entities = args.entities.map(this.replaceTypeAndId); } return await this.destroy(args.entity ?? args.entities); } private async countAndWrapValueFromcapabilityArgs(args: Record<string, any>): Promise<NodeObject> { const count = await this.count(args); return { [EngineConstants.dataSource.countResult]: { [PROP_ENTITY_VALUE]: count, [PROP_ENTITY_TYPE]: XSD.integer } }; } private async existsAndWrapValueFromcapabilityArgs(args: Record<string, any>): Promise<NodeObject> { const exists = await this.exists(args); return { [EngineConstants.dataSource.existsResult]: { [PROP_ENTITY_VALUE]: exists, [PROP_ENTITY_TYPE]: XSD.boolean } }; } public async findAndExecuteCapability( capabilityId: string, args: Record<string, any>, capabilityConfig?: CapabilityConfig ): Promise<OrArray<NodeObject>> { const capability = (await this.findBy({ id: capabilityId })) as Capability; return await this.executeCapability(capability, args, capabilityConfig); } private async executeParallelMapping( mapping: MappingWithParallel, args: JSONObject, capabilityConfig?: CapabilityConfig ): Promise<NodeObject[]> { const parallelCapabilityMappings = ensureArray( mapping[EngineConstants.prop.parallel] as unknown as OrArray<CapabilityMapping> ); const nestedReturnValues = await Promise.all<Promise<OrArray<NodeObject>>>( parallelCapabilityMappings.map( (capabilityMapping): Promise<OrArray<NodeObject>> => this.executeMapping(capabilityMapping, args, capabilityConfig) ) ); return nestedReturnValues.flat(); } private async findCapabilityIntegrationMapping( capabilityId: string, integratedProductId: string ): Promise<CapabilityMapping | undefined> { return (await this.findByIfExists({ type: EngineConstants.spec.capabilityMapping, [EngineConstants.prop.capability]: capabilityId, [EngineConstants.prop.integration]: integratedProductId })) as CapabilityMapping; } private async performOperationMappingWithArgs( args: JSONValue, mapping: Mapping, capabilityConfig?: CapabilityConfig ): Promise<NodeObject> { if (mapping[EngineConstants.prop.operationId]) { return { [EngineConstants.prop.operationId]: mapping[EngineConstants.prop.operationId] }; } if (mapping[EngineConstants.prop.dataSource]) { return { [EngineConstants.prop.dataSource]: mapping[EngineConstants.prop.dataSource] }; } return await this.performMapping( args, mapping[EngineConstants.prop.operationMapping] as OrArray<NodeObject>, undefined, capabilityConfig ); } private async performOperation( operationInfo: NodeObject, operationArgs: JSONObject, originalArgs: JSONObject, account: Entity, capabilityConfig?: CapabilityConfig, securityCredentials?: Entity ): Promise<OperationResponse> { if (operationInfo[EngineConstants.prop.schemeName]) { return await this.performOauthSecuritySchemeStageWithCredentials( operationInfo, operationArgs, account, securityCredentials ); } if (operationInfo[EngineConstants.prop.dataSource]) { return await this.getDataFromDataSource( getIdFromNodeObjectIfDefined(operationInfo[EngineConstants.prop.dataSource] as string | ReferenceNodeObject)!, capabilityConfig ); } if (operationInfo[EngineConstants.prop.operationId]) { const response = await this.performOpenapiOperationWithCredentials( getValueIfDefined(operationInfo[EngineConstants.prop.operationId])!, operationArgs, account, capabilityConfig ); return this.axiosResponseAndParamsToOperationResponse(response, operationArgs, originalArgs); } throw new Error('Operation not supported.'); } private axiosResponseAndParamsToOperationResponse( response: AxiosResponse, operationParameters: JSONObject, originalArgs: JSONObject ): OperationResponse { return { operationParameters, originalCapabilityParameters: originalArgs, data: response.data, status: response.status, statusText: response.statusText, headers: response.headers, config: { headers: response.config.headers, method: response.config.method, url: response.config.url, data: response.config.data } as JSONObject }; } private async performReturnValueMappingWithFrameIfDefined( returnValue: JSONValue, mapping: MappingWithOutputsMapping, capabilityConfig?: CapabilityConfig ): Promise<NodeObject> { if (EngineConstants.prop.outputsMapping in mapping) { return await this.performMapping( returnValue, mapping[EngineConstants.prop.outputsMapping], getValueIfDefined<JSONObject>(mapping[EngineConstants.prop.outputsMappingFrame]), capabilityConfig ); } return returnValue as NodeObject; } private async performParameterMappingOnArgsIfDefined( args: JSONObject, mapping: Partial<MappingWithInputs> | Partial<MappingWithInputsReference>, capabilityConfig?: CapabilityConfig, convertToJsonDeep = false ): Promise<Record<string, any>> { if (EngineConstants.prop.inputsReference in mapping) { const reference = getValueIfDefined<string>(mapping[EngineConstants.prop.inputsReference])!; return this.getDataAtReference(reference, args); } if (EngineConstants.prop.inputsMappingRef in mapping) { const reference = getValueIfDefined<string>(mapping[EngineConstants.prop.inputsMappingRef])!; const referencedMapping = this.getDataAtReference(reference, args); if (!referencedMapping || referencedMapping?.length === 0) { return args; } // Handle inputsMappingFrameRef if present let frame; if (EngineConstants.prop.inputsMappingFrameRef in mapping) { const frameReference = getValueIfDefined<string>(mapping[EngineConstants.prop.inputsMappingFrameRef])!; frame = this.getDataAtReference(frameReference, args); } else { // Use direct frame if provided frame = getValueIfDefined(mapping[EngineConstants.prop.inputsMappingFrame]); } // Perform mapping with the referenced mapping and frame const mappedData = await this.performMapping(args, referencedMapping, frame, capabilityConfig); return toJSON(mappedData, convertToJsonDeep); } if (EngineConstants.prop.inputsMapping in mapping) { const mappedData = await this.performMapping( args, (mapping as MappingWithInputs)[EngineConstants.prop.inputsMapping]!, getValueIfDefined(mapping[EngineConstants.prop.inputsMappingFrame]), capabilityConfig ); return toJSON(mappedData, convertToJsonDeep); } return args; } private getDataAtReference(reference: string, data: JSONObject): any { const results = JSONPath({ path: reference, json: data, resultType: 'value' }); const isArrayOfLengthOne = Array.isArray(results) && results.length === 1; let result = isArrayOfLengthOne ? results[0] : results; if (result && typeof result === 'object' && PROP_ENTITY_VALUE in result) { result = result[PROP_ENTITY_VALUE]; } return result; } private async getIntegrationInterface( integratedProductId: string, integrationType = EngineConstants.spec.integrationInterface ): Promise<Entity | null> { if (integrationType === EngineConstants.spec.integrationInterface) { const integrationInterface = await this.findBy({ type: integrationType, [EngineConstants.prop.type]: EngineConstants.spec.openApi, [EngineConstants.prop.integration]: integratedProductId }); return integrationInterface; } // Add support for other integration types return null; } private async findSecurityCredentialsForAccountIfDefined(accountId: string): Promise<Entity | undefined> { return await this.findByIfExists({ type: EngineConstants.spec.integrationAuthenticationCredential, [EngineConstants.prop.accountOrUser]: accountId }); } private async findgetOpenApiRuntimeAuthorizationCapabilityIfDefined(): Promise<Capability | undefined> { return (await this.findByIfExists({ type: EngineConstants.spec.capability, [EngineConstants.prop.label]: OPEN_API_RUNTIME_AUTHORIZATION })) as Capability; } private async getRuntimeCredentialsWithSecurityCredentials( securityCredentials: Entity, integrationId: string, openApiOperationInformation: OperationWithPathInfo, operationArgs: JSONObject ): Promise<JSONObject> { const getOpenApiRuntimeAuthorizationCapability = await this.findgetOpenApiRuntimeAuthorizationCapabilityIfDefined(); if (!getOpenApiRuntimeAuthorizationCapability) { return {}; } const mapping = await this.findCapabilityIntegrationMapping( getOpenApiRuntimeAuthorizationCapability[PROP_ENTITY_ID], integrationId ); if (!mapping) { return {}; } const args = { securityCredentials, openApiExecutorOperationWithPathInfo: openApiOperationInformation, operationArgs } as JSONObject; const operationInfoJsonLd = await this.performParameterMappingOnArgsIfDefined(args, mapping, undefined, true); const headers = getValueIfDefined<JSONObject>(operationInfoJsonLd[EngineConstants.prop.headers]); return headers ?? {}; } private async createOpenApiOperationExecutorWithSpec(openApiDescription: OpenApi): Promise<OpenApiOperationExecutor> { const executor = new OpenApiOperationExecutor(); await executor.setOpenapiSpec(openApiDescription); return executor; } private async findCapabilityObjectMapping(capabilityId: string, object: string): Promise<CapabilityMapping> { return (await this.findByIfExists({ type: EngineConstants.spec.capabilityMapping, [EngineConstants.prop.capability]: capabilityId, [EngineConstants.prop.object]: InversePath({ subPath: ZeroOrMorePath({ subPath: RDFS.subClassOf as string }), value: object }) })) as CapabilityMapping; } private async performCapabilityMappingWithArgs( args: JSONValue, mapping: Mapping, capabilityConfig?: CapabilityConfig ): Promise<string | undefined> { if (mapping[EngineConstants.prop.operationId]) { return getValueIfDefined<string>(mapping[EngineConstants.prop.operationId])!; } if (mapping[EngineConstants.prop.operationId]) { return getValueIfDefined<string>(mapping[EngineConstants.prop.operationId])!; } const capabilityInfoJsonLd = await this.performMapping( args, mapping[EngineConstants.prop.operationMapping] as NodeObject, undefined, capabilityConfig ); return getValueIfDefined<string>(capabilityInfoJsonLd[EngineConstants.prop.operationId])!; } private async assertCapabilityParamsMatchParameterSchemas( capabilityParams: any, capability: Capability ): Promise<void> { let parametersSchemaObject = capability[EngineConstants.prop.inputs]; if (parametersSchemaObject?.[PROP_ENTITY_ID] && Object.keys(parametersSchemaObject).length === 1) { parametersSchemaObject = await this.findBy({ id: parametersSchemaObject[PROP_ENTITY_ID] }); } if (capabilityParams && parametersSchemaObject) { const capabilityParamsAsJsonLd = { '@context': getValueIfDefined<ContextDefinition>(capability[EngineConstants.prop.inputsContext]), [PROP_ENTITY_TYPE]: EngineConstants.spec.inputs, ...capabilityParams }; const report = await this.convertToQuadsAndValidateAgainstShape(capabilityParamsAsJsonLd, parametersSchemaObject); if (!report.conforms) { this.throwValidationReportError( report, `${getValueIfDefined(capability[EngineConstants.prop.label])} parameters do not conform to the schema` ); } } } private async performOpenapiOperationWithCredentials( operationId: string, operationA