@comake/skl-js-engine
Version:
Standard Knowledge Language Javascript Engine
1,267 lines (1,163 loc) • 72.7 kB
text/typescript
/* eslint-disable no-div-regex */
/* eslint-disable line-comment-position */
/* eslint-disable no-inline-comments */
/* eslint-disable require-unicode-regexp */
/* eslint-disable @typescript-eslint/naming-convention */
import type { OpenApi, OpenApiClientConfiguration, OperationWithPathInfo } from '@comake/openapi-operation-executor';
import { OpenApiOperationExecutor } from '@comake/openapi-operation-executor';
import { getIdFromNodeObjectIfDefined, XSD, type ReferenceNodeObject } from '@comake/rmlmapper-js';
import type { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios';
import axios from 'axios';
import type { ContextDefinition, GraphObject, NodeObject } from 'jsonld';
import type { Frame } from 'jsonld/jsonld-spec';
import { JSONPath } from 'jsonpath-plus';
import SHACLValidator from 'rdf-validate-shacl';
import type ValidationReport from 'rdf-validate-shacl/src/validation-report';
import {
EngineConstants,
OPEN_API_RUNTIME_AUTHORIZATION,
PROP_ENTITY_ID,
PROP_ENTITY_TYPE,
PROP_ENTITY_VALUE,
RDF,
RDFS,
RML_LIST,
SHACL
} from './constants';
import { globalCustomCapabilities } from './customCapabilities';
import { globalHooks, HookStages, HookTypes } from './hooks/globalHooks';
import type { ExecutionOptions } from './JsExecutor';
import type { ICodeExecutor } from './JsExecutor/types';
import { Logger } from './logger';
import { Mapper } from './mapping/Mapper';
import type { SklEngineOptions } from './SklEngineOptions';
import type { FindOperator } from './storage/FindOperator';
import type { FindAllOptions, FindOneOptions, FindOptionsWhere } from './storage/FindOptionsTypes';
import type { GroupByOptions, GroupByResponse } from './storage/GroupOptionTypes';
import { Exists } from './storage/operator/Exists';
import { In } from './storage/operator/In';
import { InversePath } from './storage/operator/InversePath';
import { Not } from './storage/operator/Not';
import { OneOrMorePath } from './storage/operator/OneOrMorePath';
import { SequencePath } from './storage/operator/SequencePath';
import { ZeroOrMorePath } from './storage/operator/ZeroOrMorePath';
import type { QueryAdapter, RawQueryResult } from './storage/query-adapter/QueryAdapter';
import { SparqlQueryAdapter } from './storage/query-adapter/sparql/SparqlQueryAdapter';
import { PerformanceLogger } from './util/PerformanceLogger';
// Import './util/safeJsonStringify';
import type {
Callbacks,
Capability,
CapabilityConfig,
CapabilityMapping,
Entity,
JSONObject,
JSONValue,
Mapping,
MappingWithInputs,
MappingWithInputsReference,
MappingWithOutputsMapping,
MappingWithParallel,
MappingWithSeries,
OperationResponse,
OrArray,
RdfList,
SeriesCapabilityArgs,
SKLEngineInterface,
TriggerMapping
} from './util/Types';
import { convertJsonLdToQuads, ensureArray, getValueIfDefined, toJSON } from './util/Util';
import { SKL_DATA_NAMESPACE, SKL_NAMESPACE } from './util/Vocabularies';
export type CapabilityHandler = <T extends OrArray<NodeObject> = OrArray<NodeObject>>(
params: JSONObject,
capabilityConfig?: CapabilityConfig
) => Promise<T>;
export type CapabilityInterface = Record<string, CapabilityHandler>;
export type MappingResponseOption<T extends boolean> = T extends true ? JSONObject : NodeObject;
export type WriteOptions = { bypassHooks?: boolean };
export class SKLEngine implements SKLEngineInterface {
private readonly queryAdapter: QueryAdapter;
private readonly functions?: Record<string, (args: any | any[]) => any>;
private readonly inputFiles?: Record<string, string>;
private readonly globalCallbacks?: Callbacks;
private readonly disableValidation?: boolean;
public readonly capability: CapabilityInterface;
private readonly isDebugMode: boolean;
private codeExecutor: ICodeExecutor | undefined;
private readonly skdsEndpointUrl: string;
private readonly scriptPath: string;
public constructor(options: SklEngineOptions) {
this.queryAdapter = new SparqlQueryAdapter(options);
this.disableValidation = options.disableValidation;
this.globalCallbacks = options.callbacks;
this.inputFiles = options.inputFiles;
this.skdsEndpointUrl = (options as any).endpointUrl;
this.scriptPath = (options as any).scriptPath;
if (options.functions) {
this.functions = Object.fromEntries(
Object.entries(options.functions).map(([key, func]) => [
key,
(data: Record<string | number, any> = {}) => {
// Add the SKL instance to the data object
// eslint-disable-next-line @typescript-eslint/no-this-alias
data.skl = this;
// Call the original function
return func(data);
}
])
);
}
this.isDebugMode = options.debugMode ?? false;
Logger.getInstance(this.isDebugMode);
// eslint-disable-next-line func-style
const getCapabilityHandler =
(getTarget: CapabilityInterface, property: string): CapabilityHandler =>
async <T extends OrArray<NodeObject> = OrArray<NodeObject>>(
capabilityArgs: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<T> =>
this.executeCapabilityByName(property, capabilityArgs, capabilityConfig) as Promise<T>;
this.capability = new Proxy({} as CapabilityInterface, { get: getCapabilityHandler });
}
public setCodeExecutor(codeExecutor: ICodeExecutor): void {
this.codeExecutor = codeExecutor;
}
public async executeRawQuery<T extends RawQueryResult>(query: string): Promise<T[]> {
return await this.queryAdapter.executeRawQuery<T>(query);
}
public async executeRawUpdate(query: string): Promise<void> {
return await this.queryAdapter.executeRawUpdate(query);
}
public async executeRawConstructQuery(query: string, frame?: Frame): Promise<GraphObject> {
return await this.queryAdapter.executeRawConstructQuery(query, frame);
}
public async find(options?: FindOneOptions): Promise<Entity> {
return PerformanceLogger.withSpanRoot('SklEngine.find', async() => {
const context = {
entities: [],
operation: 'find',
operationParameters: { options },
sklEngine: this
};
await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context);
try {
const entity = await this.queryAdapter.find(options);
if (!entity) {
throw new Error(`No schema found with fields matching ${JSON.stringify(options)}`);
}
const updatedContext = { ...context, entities: [entity]};
const afterHookResult = await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entity);
return afterHookResult || entity;
} catch (error: unknown) {
await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error);
throw error;
}
}, { options });
}
public async findBy(where: FindOptionsWhere, notFoundErrorMessage?: string): Promise<Entity> {
return PerformanceLogger.withSpanRoot('SklEngine.findBy', async() => {
const context = {
entities: [],
operation: 'findBy',
operationParameters: { where },
sklEngine: this
};
await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context);
try {
const entity = await this.queryAdapter.findBy(where);
if (entity) {
const updatedContext = { ...context, entities: [entity]};
await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entity);
return entity;
}
throw new Error(notFoundErrorMessage ?? `No schema found with fields matching ${JSON.stringify(where)}`);
} catch (error: unknown) {
await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error);
throw error;
}
}, { where });
}
public async findByIfExists(options: FindOptionsWhere): Promise<Entity | undefined> {
try {
const entity = await this.findBy(options);
return entity;
} catch {
return undefined;
}
}
public async findAll(options?: FindAllOptions): Promise<Entity[]> {
return PerformanceLogger.withSpanRoot('SklEngine.findAll', async() => {
const context = {
entities: [],
operation: 'findAll',
operationParameters: { options },
sklEngine: this
};
await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context);
try {
const entities = await this.queryAdapter.findAll(options);
const updatedContext = { ...context, entities };
await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entities);
return entities;
} catch (error: unknown) {
await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error);
throw error;
}
}, { options });
}
public async groupBy(options: GroupByOptions): Promise<GroupByResponse> {
return PerformanceLogger.withSpanRoot('SklEngine.groupBy', async() => {
const context = {
entities: [],
operation: 'groupBy',
operationParameters: { options },
sklEngine: this
};
await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context);
try {
const result = await this.queryAdapter.groupBy(options);
const updatedContext = { ...context, result };
await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, result);
return result;
} catch (error: unknown) {
await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error);
throw error;
}
}, { options });
}
public async findAllBy(where: FindOptionsWhere): Promise<Entity[]> {
return PerformanceLogger.withSpanRoot('SklEngine.findAllBy', async() => {
const context = {
entities: [],
operation: 'findAllBy',
operationParameters: { where },
sklEngine: this
};
await globalHooks.execute(HookTypes.READ, HookStages.BEFORE, context);
try {
const entities = await this.queryAdapter.findAllBy(where);
const updatedContext = { ...context, entities };
await globalHooks.execute(HookTypes.READ, HookStages.AFTER, updatedContext, entities);
return entities;
} catch (error: unknown) {
await globalHooks.execute(HookTypes.READ, HookStages.ERROR, context, error);
throw error;
}
}, { where });
}
public async exists(options?: FindAllOptions): Promise<boolean> {
return PerformanceLogger.withSpanRoot('SklEngine.exists', async() => this.queryAdapter.exists(options), { options });
}
public async count(options?: FindAllOptions): Promise<number> {
return PerformanceLogger.withSpanRoot('SklEngine.count', async() => this.queryAdapter.count(options), { options });
}
public async save(entity: Entity, options?: WriteOptions): Promise<Entity>;
public async save(entities: Entity[], options?: WriteOptions): Promise<Entity[]>;
public async save(entityOrEntities: Entity | Entity[], options?: WriteOptions): Promise<Entity | Entity[]> {
return PerformanceLogger.withSpanRoot('SklEngine.save', async() => {
const entityArray = Array.isArray(entityOrEntities) ? entityOrEntities : [entityOrEntities];
const isSingleEntity = !Array.isArray(entityOrEntities);
await globalHooks.executeBeforeCreate(entityArray, { sklEngine: this, bypassHooks: options?.bypassHooks });
try {
await this.validateEntitiesConformToObjectSchema(entityArray);
const savedEntities = await this.queryAdapter.save(entityArray);
await globalHooks.executeAfterCreate(savedEntities, { sklEngine: this, bypassHooks: options?.bypassHooks });
return isSingleEntity ? savedEntities[0] : savedEntities;
} catch (error) {
await globalHooks.executeErrorCreate(entityArray, error as Error, { sklEngine: this, bypassHooks: options?.bypassHooks });
throw error;
}
}, { entityCount: Array.isArray(entityOrEntities) ? entityOrEntities.length : 1 });
}
public async update(id: string, attributes: Partial<Entity>, options?: WriteOptions): Promise<void>;
public async update(ids: string[], attributes: Partial<Entity>, options?: WriteOptions): Promise<void>;
public async update(idOrIds: string | string[], attributes: Partial<Entity>, options?: WriteOptions): Promise<void> {
return PerformanceLogger.withSpanRoot('SklEngine.update', async() => {
const idArray = Array.isArray(idOrIds) ? idOrIds : [idOrIds];
const isSingleEntity = !Array.isArray(idOrIds);
await globalHooks.execute(HookTypes.UPDATE, HookStages.BEFORE, {
entities: [],
operation: 'update',
operationParameters: { idArray, attributes },
sklEngine: this,
bypassHooks: options?.bypassHooks
});
try {
if (idArray.length > 1) {
await this.validateEntitiesWithIdsConformsToObjectSchemaForAttributes(idArray, attributes);
} else {
await this.validateEntityWithIdConformsToObjectSchemaForAttributes(idArray[0], attributes);
}
await this.queryAdapter.update(isSingleEntity ? idArray[0] : (idArray as any), attributes);
await globalHooks.execute(HookTypes.UPDATE, HookStages.AFTER, {
entities: [],
operation: 'update',
operationParameters: { idArray, attributes },
sklEngine: this,
bypassHooks: options?.bypassHooks
});
} catch (error: unknown) {
await globalHooks.execute(HookTypes.UPDATE, HookStages.ERROR, {
entities: [],
operation: 'update',
operationParameters: { idArray, attributes },
sklEngine: this,
bypassHooks: options?.bypassHooks
}, error);
throw error;
}
}, { idCount: Array.isArray(idOrIds) ? idOrIds.length : 1 });
}
public async validateEntitiesConformToObjectSchema(entities: Entity[]): Promise<void> {
const entitiesByType = this.groupEntitiesByType(entities);
for (const type of Object.keys(entitiesByType)) {
const object = await this.findByIfExists({ id: type });
if (object) {
const parentObjects = await this.getSuperClassesOfObject(type);
for (const currentObject of [object, ...parentObjects]) {
const entitiesOfType = entitiesByType[type];
const nounSchemaWithTarget = {
...currentObject,
[SHACL.targetNode]: entitiesOfType.map(
(entity): ReferenceNodeObject => ({ [PROP_ENTITY_ID]: entity[PROP_ENTITY_ID] })
)
};
const report = await this.convertToQuadsAndValidateAgainstShape(entitiesOfType, nounSchemaWithTarget);
if (!report.conforms) {
const entityIds = entitiesOfType.map((entity): string => entity[PROP_ENTITY_ID]);
this.throwValidationReportError(
report,
`Entity ${entityIds.join(', ')} does not conform to the ${currentObject[PROP_ENTITY_ID]} schema.`
);
}
}
}
}
}
private groupEntitiesByType(entities: Entity[]): Record<string, Entity[]> {
return entities.reduce((groupedEntities: Record<string, Entity[]>, entity): Record<string, Entity[]> => {
const entityTypes = Array.isArray(entity[PROP_ENTITY_TYPE])
? entity[PROP_ENTITY_TYPE]
: [entity[PROP_ENTITY_TYPE]];
for (const type of entityTypes) {
if (!groupedEntities[type]) {
groupedEntities[type] = [];
}
groupedEntities[type].push(entity);
}
return groupedEntities;
}, {});
}
private async getSuperClassesOfObject(object: string): Promise<Entity[]> {
return await this.getParentsOfSelector(object);
}
private async getSuperClassesOfObjects(nouns: string[]): Promise<Entity[]> {
return await this.getParentsOfSelector(In(nouns));
}
private async getParentsOfSelector(selector: string | FindOperator<any, any>): Promise<Entity[]> {
return await this.findAll({
where: {
id: InversePath({
subPath: OneOrMorePath({ subPath: RDFS.subClassOf as string }),
value: selector
})
}
});
}
private async validateEntityConformsToObjectSchema(entity: Entity): Promise<void> {
const nounIds = Array.isArray(entity[PROP_ENTITY_TYPE]) ? entity[PROP_ENTITY_TYPE] : [entity[PROP_ENTITY_TYPE]];
const directObjects = await this.findAllBy({ id: In(nounIds) });
if (directObjects.length > 0) {
const existingObjectIds = directObjects.map((object): string => object[PROP_ENTITY_ID]);
const parentObjects = await this.getSuperClassesOfObjects(existingObjectIds);
for (const currentObject of [...directObjects, ...parentObjects]) {
const nounSchemaWithTarget = {
...currentObject,
[SHACL.targetNode]: { [PROP_ENTITY_ID]: entity[PROP_ENTITY_ID] }
};
const report = await this.convertToQuadsAndValidateAgainstShape(entity, nounSchemaWithTarget);
if (!report.conforms) {
this.throwValidationReportError(
report,
`Entity ${entity[PROP_ENTITY_ID]} does not conform to the ${currentObject[PROP_ENTITY_ID]} schema.`
);
}
}
}
}
private async validateEntitiesWithIdsConformsToObjectSchemaForAttributes(
ids: string[],
attributes: Partial<Entity>
): Promise<void> {
for (const id of ids) {
await this.validateEntityWithIdConformsToObjectSchemaForAttributes(id, attributes);
}
}
private async getObjectsAndParentObjectsOfEntity(id: string): Promise<Entity[]> {
return await this.findAllBy({
id: InversePath({
subPath: SequencePath({
subPath: [RDF.type, ZeroOrMorePath({ subPath: RDFS.subClassOf as string })]
}),
value: id
})
});
}
private async validateEntityWithIdConformsToObjectSchemaForAttributes(
id: string,
attributes: Partial<Entity>
): Promise<void> {
const nouns = await this.getObjectsAndParentObjectsOfEntity(id);
for (const currentObject of nouns) {
if (SHACL.property in currentObject) {
const nounProperties = ensureArray(currentObject[SHACL.property] as OrArray<NodeObject>).filter(
(property): boolean => {
const path = property[SHACL.path];
if (typeof path === 'string' && path in attributes) {
return true;
}
if (typeof path === 'object' && PROP_ENTITY_ID in path! && (path[PROP_ENTITY_ID] as string) in attributes) {
return true;
}
return false;
}
);
if (nounProperties.length > 0) {
const nounSchemaWithTarget = {
[PROP_ENTITY_TYPE]: SHACL.NodeShape,
[SHACL.targetNode]: { [PROP_ENTITY_ID]: id },
[SHACL.property]: nounProperties
};
const attributesWithId = { ...attributes, [PROP_ENTITY_ID]: id };
const report = await this.convertToQuadsAndValidateAgainstShape(attributesWithId, nounSchemaWithTarget);
if (!report.conforms) {
this.throwValidationReportError(
report,
`Entity ${id} does not conform to the ${currentObject[PROP_ENTITY_ID]} schema.`
);
}
}
}
}
}
public async delete(id: string, options?: WriteOptions): Promise<void>;
public async delete(ids: string[], options?: WriteOptions): Promise<void>;
public async delete(idOrIds: string | string[], options?: WriteOptions): Promise<void> {
return PerformanceLogger.withSpanRoot('SklEngine.delete', async() => {
const idArray = Array.isArray(idOrIds) ? idOrIds : [idOrIds];
await globalHooks.execute(HookTypes.DELETE, HookStages.BEFORE, {
entities: [],
operation: 'delete',
operationParameters: { idArray },
sklEngine: this,
bypassHooks: options?.bypassHooks
});
try {
await this.queryAdapter.delete(idArray);
await globalHooks.execute(HookTypes.DELETE, HookStages.AFTER, {
entities: [],
operation: 'delete',
operationParameters: { idArray },
sklEngine: this,
bypassHooks: options?.bypassHooks
});
} catch (error) {
await globalHooks.execute(HookTypes.DELETE, HookStages.ERROR, {
entities: [],
operation: 'delete',
operationParameters: { idArray },
sklEngine: this,
bypassHooks: options?.bypassHooks
}, error);
throw error;
}
}, { idCount: Array.isArray(idOrIds) ? idOrIds.length : 1 });
}
public async destroy(entity: Entity): Promise<Entity>;
public async destroy(entities: Entity[]): Promise<Entity[]>;
public async destroy(entityOrEntities: Entity | Entity[]): Promise<Entity | Entity[]> {
if (Array.isArray(entityOrEntities)) {
return await this.queryAdapter.destroy(entityOrEntities);
}
return await this.queryAdapter.destroy(entityOrEntities);
}
public async destroyAll(): Promise<void> {
return await this.queryAdapter.destroyAll();
}
public async performMapping(
args: JSONValue,
mapping: OrArray<NodeObject>,
frame?: Record<string, any>,
capabilityConfig?: CapabilityConfig,
jsExecutionOptions?: ExecutionOptions
): Promise<NodeObject> {
const mappingArray = ensureArray(mapping);
const codeBlocks = mappingArray.filter(
(mappingItem: NodeObject): boolean =>
mappingItem[PROP_ENTITY_TYPE] === EngineConstants.spec.codeBlock &&
this.isJavaScriptCode(getValueIfDefined(mappingItem[EngineConstants.prop.codeBody])!)
);
// FIXME: Handle if we can combine the codeb blocks with triples map.
// As of now if there is any code block, triples map does not get executed.
if (codeBlocks.length > 0) {
return await this.executeCodeBlocks(codeBlocks, args, jsExecutionOptions ?? {});
}
const functions = {
...this.functions,
...capabilityConfig?.functions
};
const mapper = new Mapper({ functions });
return await mapper.apply(args, mapping, frame ?? {});
}
public async executeTrigger(integration: string, payload: any): Promise<void> {
const triggerToCapabilityMapping = await this.findTriggerCapabilityMapping(integration);
const capabilityArgs = await this.performParameterMappingOnArgsIfDefined(
payload,
triggerToCapabilityMapping as Partial<MappingWithInputs> | Partial<MappingWithInputsReference>
);
const capabilityId = await this.performCapabilityMappingWithArgs(payload, triggerToCapabilityMapping);
if (capabilityId) {
const mappedCapability = (await this.findBy({ id: capabilityId })) as Capability;
await this.executeCapability(mappedCapability, capabilityArgs);
}
}
private async findTriggerCapabilityMapping(integration: string): Promise<TriggerMapping> {
const triggerCapabilityMappingNew = (await this.findBy(
{
type: EngineConstants.spec.capabilityMapping,
[EngineConstants.prop.capability]: integration,
[EngineConstants.prop.capabilityType]: EngineConstants.spec.triggerCapabilityMapping
},
`Failed to find a Trigger Capability mapping for integration ${integration}`
)) as TriggerMapping;
if (triggerCapabilityMappingNew) {
return triggerCapabilityMappingNew;
}
throw new Error(`Failed to find a Trigger Capability mapping for integration ${integration}`);
}
private async executeCapabilityByName(
capabilityName: string,
capabilityArgs: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<OrArray<NodeObject>> {
const capability = await this.findCapabilityWithName(capabilityName);
return await this.executeCapability(capability, capabilityArgs, capabilityConfig);
}
private async findCapabilityWithName(capabilityName: string): Promise<Capability> {
return (await this.findBy(
{ type: EngineConstants.spec.capability, [EngineConstants.prop.label]: capabilityName },
`Failed to find the capability ${capabilityName} in the schema.`
)) as Capability;
}
public async executeCapability(
capability: Capability,
capabilityArgs: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<OrArray<NodeObject>> {
this.globalCallbacks?.onCapabilityStart?.(capability[PROP_ENTITY_ID], capabilityArgs);
if (capabilityConfig?.callbacks?.onCapabilityStart) {
Logger.getInstance().log('Capability arguments', capabilityArgs);
capabilityConfig.callbacks.onCapabilityStart(capability[PROP_ENTITY_ID], capabilityArgs);
}
const { mapping, account } = await this.findMappingForCapabilityContextually(
capability[PROP_ENTITY_ID],
capabilityArgs
);
Logger.getInstance().log('Mapping', JSON.stringify(mapping));
const shouldValidate = this.shouldValidate(capabilityConfig);
if (shouldValidate) {
await this.assertCapabilityParamsMatchParameterSchemas(capabilityArgs, capability);
}
try {
// Execute capability mapping before hook if appropriate -
// works for any mapping that can be used as a verb mapping
if (mapping) {
await globalHooks.executeBeforeExecuteCapabilityMapping([capabilityArgs] as Entity[], mapping, { sklEngine: this });
}
const verbReturnValue = await this.executeMapping(mapping, capabilityArgs, capabilityConfig, account);
if (shouldValidate) {
await this.assertCapabilityReturnValueMatchesReturnTypeSchema(verbReturnValue, capability);
}
// Execute capability mapping after hook if appropriate
if (mapping) {
await globalHooks.executeAfterExecuteCapabilityMapping([capabilityArgs] as Entity[], mapping, verbReturnValue, { sklEngine: this });
}
this.globalCallbacks?.onCapabilityEnd?.(capability[PROP_ENTITY_ID], verbReturnValue);
if (capabilityConfig?.callbacks?.onCapabilityEnd) {
capabilityConfig.callbacks.onCapabilityEnd(capability[PROP_ENTITY_ID], verbReturnValue);
}
return verbReturnValue;
} catch (error) {
// Execute capability mapping error hook if appropriate
if (mapping) {
await globalHooks.executeErrorExecuteCapabilityMapping([capabilityArgs] as Entity[], mapping, error as Error, { sklEngine: this });
}
throw error;
}
}
private async findMappingForCapabilityContextually(
capabilityId: string,
args: JSONObject
): Promise<{ mapping: CapabilityMapping; account?: Entity }> {
if (args.mapping) {
const mapping = await this.findByIfExists({ id: args.mapping as string });
if (!mapping) {
throw new Error(`Mapping ${args.mapping as string} not found.`);
}
return { mapping: mapping as CapabilityMapping };
}
if (args.object) {
const mapping = await this.findCapabilityObjectMapping(capabilityId, args.object as string);
if (mapping) {
return { mapping };
}
}
if (args.account) {
const account = await this.findBy({ id: args.account as string });
const integratedProductId = (account[EngineConstants.prop.integration] as ReferenceNodeObject)[PROP_ENTITY_ID];
const mapping = await this.findCapabilityIntegrationMapping(capabilityId, integratedProductId);
if (mapping) {
return { mapping, account };
}
}
const mappings = await this.findAllBy({
type: EngineConstants.spec.capabilityMapping,
[EngineConstants.prop.capability]: capabilityId,
[EngineConstants.prop.integration]: Not(Exists()),
[EngineConstants.prop.object]: Not(Exists())
});
if (mappings.length === 1) {
return { mapping: mappings[0] as CapabilityMapping };
}
if (mappings.length > 1) {
throw new Error('Multiple mappings found for capability, please specify one.');
}
if (args.object) {
throw new Error(`Mapping between object ${args.object as string} and capability ${capabilityId} not found.`);
}
if (args.account) {
throw new Error(`Mapping between account ${args.account as string} and capability ${capabilityId} not found.`);
}
throw new Error(`No mapping found.`);
}
public async executeMapping(
mapping: Mapping,
args: JSONObject,
capabilityConfig?: CapabilityConfig,
account?: Entity
): Promise<OrArray<NodeObject>> {
args = await this.addPreProcessingMappingToArgs(mapping, args, capabilityConfig);
let returnValue: OrArray<NodeObject>;
// If (EngineConstants.prop.capability in mapping || EngineConstants.prop.capabilityMapping in mapping) {
// const capabilityId = await this.performCapabilityMappingWithArgs(args, mapping, capabilityConfig);
// const mappedArgs = await this.performParameterMappingOnArgsIfDefined(
// { ...args, capabilityId },
// mapping as MappingWithInputs,
// capabilityConfig
// );
// Logger.getInstance().log('Mapped args', mappedArgs);
// returnValue = await this.executeCapabilityMapping(mapping, args, mappedArgs, capabilityConfig);
// } else {
const mappedArgs = await this.performParameterMappingOnArgsIfDefined(
args,
mapping as MappingWithInputs,
capabilityConfig
);
Logger.getInstance().log('Mapped args', mappedArgs);
if (EngineConstants.prop.operationId in mapping || EngineConstants.prop.operationMapping in mapping) {
returnValue = (await this.executeOperationMapping(
mapping,
mappedArgs,
args,
account!,
capabilityConfig
)) as NodeObject;
} else if (EngineConstants.prop.series in mapping) {
returnValue = await this.executeSeriesMapping(mapping as MappingWithSeries, mappedArgs, capabilityConfig);
} else if (EngineConstants.prop.parallel in mapping) {
returnValue = await this.executeParallelMapping(mapping as MappingWithParallel, mappedArgs, capabilityConfig);
} else {
returnValue = mappedArgs;
}
// }
return await this.performReturnValueMappingWithFrameIfDefined(
returnValue as JSONValue,
mapping as MappingWithOutputsMapping,
capabilityConfig
);
}
private shouldValidate(capabilityConfig?: CapabilityConfig): boolean {
return capabilityConfig?.disableValidation === undefined
? this.disableValidation !== true
: !capabilityConfig.disableValidation;
}
private async executeOperationMapping(
mapping: Mapping,
mappedArgs: JSONObject,
originalArgs: JSONObject,
account: Entity,
capabilityConfig?: CapabilityConfig
): Promise<OperationResponse | OrArray<NodeObject>> {
const integration = (mapping as CapabilityMapping)[EngineConstants.prop.integration]?.[PROP_ENTITY_ID];
// If the mapping has an integration, it means that the operation is an integration operation
if (integration) {
const operationInfo = await this.performOperationMappingWithArgs(originalArgs, mapping, capabilityConfig);
const response = await this.performOperation(operationInfo, mappedArgs, originalArgs, account, capabilityConfig);
if (!this.ifCapabilityStreaming(capabilityConfig)) {
Logger.getInstance().log('Original response', JSON.stringify(response));
}
return response;
}
// If the mapping does not have an integration, it means that the operation is a capability operation
return await this.executeCapabilityMapping(mapping, originalArgs, mappedArgs, capabilityConfig);
}
private async executeSeriesMapping(
mapping: MappingWithSeries,
args: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<OrArray<NodeObject>> {
const seriesCapabilityMappingsList = this.rdfListToArray(mapping[EngineConstants.prop.series]!);
const seriesCapabilityArgs = {
originalCapabilityParameters: args,
previousCapabilityReturnValue: {},
allStepsResults: []
};
return await this.executeSeriesFromList(seriesCapabilityMappingsList, seriesCapabilityArgs, capabilityConfig);
}
private rdfListToArray(list: { [RML_LIST]: CapabilityMapping[] } | RdfList<CapabilityMapping>): CapabilityMapping[] {
if (!(RML_LIST in list)) {
return [
list[RDF.first],
...getIdFromNodeObjectIfDefined(list[RDF.rest] as ReferenceNodeObject) === RDF.nil
? []
: this.rdfListToArray(list[RDF.rest] as RdfList<CapabilityMapping>)
];
}
return list[RML_LIST];
}
private async executeSeriesFromList(
list: Mapping[],
args: SeriesCapabilityArgs,
capabilityConfig?: CapabilityConfig
): Promise<OrArray<NodeObject>> {
const nextCapabilityMapping = list[0];
const returnValue = await this.executeMapping(nextCapabilityMapping, args, capabilityConfig);
if (list.length > 1) {
return await this.executeSeriesFromList(
list.slice(1),
{
...args,
previousCapabilityReturnValue: returnValue as JSONObject,
allStepsResults: [...args.allStepsResults ?? [], returnValue as JSONObject]
},
capabilityConfig
);
}
return returnValue;
}
private async executeCapabilityMapping(
capabilityMapping: Mapping,
originalArgs: JSONObject,
mappedArgs: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<OrArray<NodeObject>> {
const capabilityId = await this.performCapabilityMappingWithArgs(originalArgs, capabilityMapping, capabilityConfig);
if (capabilityId) {
if (capabilityId === EngineConstants.dataSource.update) {
await this.updateEntityFromcapabilityArgs(mappedArgs);
return {};
}
if (capabilityId === EngineConstants.dataSource.save) {
return await this.saveEntityOrEntitiesFromcapabilityArgs(mappedArgs);
}
if (capabilityId === EngineConstants.dataSource.destroy) {
return await this.destroyEntityOrEntitiesFromcapabilityArgs(mappedArgs);
}
if (capabilityId === EngineConstants.dataSource.findAll) {
return await this.findAll(mappedArgs);
}
if (capabilityId === EngineConstants.dataSource.find) {
return await this.find(mappedArgs);
}
if (capabilityId === EngineConstants.dataSource.count) {
return await this.countAndWrapValueFromcapabilityArgs(mappedArgs);
}
if (capabilityId === EngineConstants.dataSource.exists) {
return await this.existsAndWrapValueFromcapabilityArgs(mappedArgs);
}
if (capabilityId === 'https://skl.ai/capability/execute-code') {
const codeBlocks = ensureArray((capabilityMapping[EngineConstants.prop.codeBlocks] as any[]) ?? []).filter(
(mappingItem: any): boolean =>
mappingItem[PROP_ENTITY_TYPE] === EngineConstants.spec.codeBlock &&
this.isJavaScriptCode(getValueIfDefined(mappingItem[EngineConstants.prop.codeBody])!)
);
return await this.executeCodeBlocks(codeBlocks, mappedArgs, {});
}
// Check for custom capabilities
if (globalCustomCapabilities.has(capabilityId)) {
return await globalCustomCapabilities.execute(capabilityId, mappedArgs, this, capabilityConfig);
}
return await this.findAndExecuteCapability(capabilityId, mappedArgs, capabilityConfig);
}
return {};
}
private async addPreProcessingMappingToArgs(
capabilityMapping: Mapping,
args: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<JSONObject> {
if (EngineConstants.prop.preProcessingMapping in capabilityMapping) {
const preMappingArgs = await this.performMapping(
args,
capabilityMapping[EngineConstants.prop.preProcessingMapping] as NodeObject,
getValueIfDefined(capabilityMapping[EngineConstants.prop.preProcessingMappingFrame]),
capabilityConfig
);
return { ...args, preProcessedParameters: preMappingArgs as JSONObject };
}
return args;
}
private replaceTypeAndId(entity: Record<string, any>): Record<string, any> {
if (typeof entity !== 'object') {
throw new Error('Entity is not an object');
}
const clonedEntity = structuredClone(entity);
if (clonedEntity[EngineConstants.prop.type]) {
clonedEntity[PROP_ENTITY_TYPE] = clonedEntity[EngineConstants.prop.type];
}
if (clonedEntity[EngineConstants.prop.identifier]) {
clonedEntity[PROP_ENTITY_ID] = SKL_DATA_NAMESPACE + (clonedEntity[EngineConstants.prop.identifier] as string);
}
return clonedEntity;
}
private async updateEntityFromcapabilityArgs(args: Record<string, any>): Promise<void> {
let ids = args.id ?? args.ids;
if (!Array.isArray(ids)) {
ids = [ids];
}
// FIX: Temporary fix for the issue where the id always getting prefixed with the namespace
ids = ids.map((id: string) => (id.startsWith('http') ? id : `${SKL_DATA_NAMESPACE}${id}`));
await this.update(ids, args.attributes);
}
private async saveEntityOrEntitiesFromcapabilityArgs(args: Record<string, any>): Promise<OrArray<Entity>> {
if (args.entity && typeof args.entity === 'object') {
args.entity = this.replaceTypeAndId(args.entity);
}
if (args.entities && Array.isArray(args.entities)) {
args.entities = args.entities.map(this.replaceTypeAndId);
}
return await this.save(args.entity ?? args.entities);
}
private async destroyEntityOrEntitiesFromcapabilityArgs(args: Record<string, any>): Promise<OrArray<Entity>> {
if (args.entity && typeof args.entity === 'object') {
args.entity = this.replaceTypeAndId(args.entity);
}
if (args.entities && Array.isArray(args.entities)) {
args.entities = args.entities.map(this.replaceTypeAndId);
}
return await this.destroy(args.entity ?? args.entities);
}
private async countAndWrapValueFromcapabilityArgs(args: Record<string, any>): Promise<NodeObject> {
const count = await this.count(args);
return {
[EngineConstants.dataSource.countResult]: {
[PROP_ENTITY_VALUE]: count,
[PROP_ENTITY_TYPE]: XSD.integer
}
};
}
private async existsAndWrapValueFromcapabilityArgs(args: Record<string, any>): Promise<NodeObject> {
const exists = await this.exists(args);
return {
[EngineConstants.dataSource.existsResult]: {
[PROP_ENTITY_VALUE]: exists,
[PROP_ENTITY_TYPE]: XSD.boolean
}
};
}
public async findAndExecuteCapability(
capabilityId: string,
args: Record<string, any>,
capabilityConfig?: CapabilityConfig
): Promise<OrArray<NodeObject>> {
const capability = (await this.findBy({ id: capabilityId })) as Capability;
return await this.executeCapability(capability, args, capabilityConfig);
}
private async executeParallelMapping(
mapping: MappingWithParallel,
args: JSONObject,
capabilityConfig?: CapabilityConfig
): Promise<NodeObject[]> {
const parallelCapabilityMappings = ensureArray(
mapping[EngineConstants.prop.parallel] as unknown as OrArray<CapabilityMapping>
);
const nestedReturnValues = await Promise.all<Promise<OrArray<NodeObject>>>(
parallelCapabilityMappings.map(
(capabilityMapping): Promise<OrArray<NodeObject>> =>
this.executeMapping(capabilityMapping, args, capabilityConfig)
)
);
return nestedReturnValues.flat();
}
private async findCapabilityIntegrationMapping(
capabilityId: string,
integratedProductId: string
): Promise<CapabilityMapping | undefined> {
return (await this.findByIfExists({
type: EngineConstants.spec.capabilityMapping,
[EngineConstants.prop.capability]: capabilityId,
[EngineConstants.prop.integration]: integratedProductId
})) as CapabilityMapping;
}
private async performOperationMappingWithArgs(
args: JSONValue,
mapping: Mapping,
capabilityConfig?: CapabilityConfig
): Promise<NodeObject> {
if (mapping[EngineConstants.prop.operationId]) {
return { [EngineConstants.prop.operationId]: mapping[EngineConstants.prop.operationId] };
}
if (mapping[EngineConstants.prop.dataSource]) {
return { [EngineConstants.prop.dataSource]: mapping[EngineConstants.prop.dataSource] };
}
return await this.performMapping(
args,
mapping[EngineConstants.prop.operationMapping] as OrArray<NodeObject>,
undefined,
capabilityConfig
);
}
private async performOperation(
operationInfo: NodeObject,
operationArgs: JSONObject,
originalArgs: JSONObject,
account: Entity,
capabilityConfig?: CapabilityConfig,
securityCredentials?: Entity
): Promise<OperationResponse> {
if (operationInfo[EngineConstants.prop.schemeName]) {
return await this.performOauthSecuritySchemeStageWithCredentials(
operationInfo,
operationArgs,
account,
securityCredentials
);
}
if (operationInfo[EngineConstants.prop.dataSource]) {
return await this.getDataFromDataSource(
getIdFromNodeObjectIfDefined(operationInfo[EngineConstants.prop.dataSource] as string | ReferenceNodeObject)!,
capabilityConfig
);
}
if (operationInfo[EngineConstants.prop.operationId]) {
const response = await this.performOpenapiOperationWithCredentials(
getValueIfDefined(operationInfo[EngineConstants.prop.operationId])!,
operationArgs,
account,
capabilityConfig
);
return this.axiosResponseAndParamsToOperationResponse(response, operationArgs, originalArgs);
}
throw new Error('Operation not supported.');
}
private axiosResponseAndParamsToOperationResponse(
response: AxiosResponse,
operationParameters: JSONObject,
originalArgs: JSONObject
): OperationResponse {
return {
operationParameters,
originalCapabilityParameters: originalArgs,
data: response.data,
status: response.status,
statusText: response.statusText,
headers: response.headers,
config: {
headers: response.config.headers,
method: response.config.method,
url: response.config.url,
data: response.config.data
} as JSONObject
};
}
private async performReturnValueMappingWithFrameIfDefined(
returnValue: JSONValue,
mapping: MappingWithOutputsMapping,
capabilityConfig?: CapabilityConfig
): Promise<NodeObject> {
if (EngineConstants.prop.outputsMapping in mapping) {
return await this.performMapping(
returnValue,
mapping[EngineConstants.prop.outputsMapping],
getValueIfDefined<JSONObject>(mapping[EngineConstants.prop.outputsMappingFrame]),
capabilityConfig
);
}
return returnValue as NodeObject;
}
private async performParameterMappingOnArgsIfDefined(
args: JSONObject,
mapping: Partial<MappingWithInputs> | Partial<MappingWithInputsReference>,
capabilityConfig?: CapabilityConfig,
convertToJsonDeep = false
): Promise<Record<string, any>> {
if (EngineConstants.prop.inputsReference in mapping) {
const reference = getValueIfDefined<string>(mapping[EngineConstants.prop.inputsReference])!;
return this.getDataAtReference(reference, args);
}
if (EngineConstants.prop.inputsMappingRef in mapping) {
const reference = getValueIfDefined<string>(mapping[EngineConstants.prop.inputsMappingRef])!;
const referencedMapping = this.getDataAtReference(reference, args);
if (!referencedMapping || referencedMapping?.length === 0) {
return args;
}
// Handle inputsMappingFrameRef if present
let frame;
if (EngineConstants.prop.inputsMappingFrameRef in mapping) {
const frameReference = getValueIfDefined<string>(mapping[EngineConstants.prop.inputsMappingFrameRef])!;
frame = this.getDataAtReference(frameReference, args);
} else {
// Use direct frame if provided
frame = getValueIfDefined(mapping[EngineConstants.prop.inputsMappingFrame]);
}
// Perform mapping with the referenced mapping and frame
const mappedData = await this.performMapping(args, referencedMapping, frame, capabilityConfig);
return toJSON(mappedData, convertToJsonDeep);
}
if (EngineConstants.prop.inputsMapping in mapping) {
const mappedData = await this.performMapping(
args,
(mapping as MappingWithInputs)[EngineConstants.prop.inputsMapping]!,
getValueIfDefined(mapping[EngineConstants.prop.inputsMappingFrame]),
capabilityConfig
);
return toJSON(mappedData, convertToJsonDeep);
}
return args;
}
private getDataAtReference(reference: string, data: JSONObject): any {
const results = JSONPath({
path: reference,
json: data,
resultType: 'value'
});
const isArrayOfLengthOne = Array.isArray(results) && results.length === 1;
let result = isArrayOfLengthOne ? results[0] : results;
if (result && typeof result === 'object' && PROP_ENTITY_VALUE in result) {
result = result[PROP_ENTITY_VALUE];
}
return result;
}
private async getIntegrationInterface(
integratedProductId: string,
integrationType = EngineConstants.spec.integrationInterface
): Promise<Entity | null> {
if (integrationType === EngineConstants.spec.integrationInterface) {
const integrationInterface = await this.findBy({
type: integrationType,
[EngineConstants.prop.type]: EngineConstants.spec.openApi,
[EngineConstants.prop.integration]: integratedProductId
});
return integrationInterface;
}
// Add support for other integration types
return null;
}
private async findSecurityCredentialsForAccountIfDefined(accountId: string): Promise<Entity | undefined> {
return await this.findByIfExists({
type: EngineConstants.spec.integrationAuthenticationCredential,
[EngineConstants.prop.accountOrUser]: accountId
});
}
private async findgetOpenApiRuntimeAuthorizationCapabilityIfDefined(): Promise<Capability | undefined> {
return (await this.findByIfExists({
type: EngineConstants.spec.capability,
[EngineConstants.prop.label]: OPEN_API_RUNTIME_AUTHORIZATION
})) as Capability;
}
private async getRuntimeCredentialsWithSecurityCredentials(
securityCredentials: Entity,
integrationId: string,
openApiOperationInformation: OperationWithPathInfo,
operationArgs: JSONObject
): Promise<JSONObject> {
const getOpenApiRuntimeAuthorizationCapability = await this.findgetOpenApiRuntimeAuthorizationCapabilityIfDefined();
if (!getOpenApiRuntimeAuthorizationCapability) {
return {};
}
const mapping = await this.findCapabilityIntegrationMapping(
getOpenApiRuntimeAuthorizationCapability[PROP_ENTITY_ID],
integrationId
);
if (!mapping) {
return {};
}
const args = {
securityCredentials,
openApiExecutorOperationWithPathInfo: openApiOperationInformation,
operationArgs
} as JSONObject;
const operationInfoJsonLd = await this.performParameterMappingOnArgsIfDefined(args, mapping, undefined, true);
const headers = getValueIfDefined<JSONObject>(operationInfoJsonLd[EngineConstants.prop.headers]);
return headers ?? {};
}
private async createOpenApiOperationExecutorWithSpec(openApiDescription: OpenApi): Promise<OpenApiOperationExecutor> {
const executor = new OpenApiOperationExecutor();
await executor.setOpenapiSpec(openApiDescription);
return executor;
}
private async findCapabilityObjectMapping(capabilityId: string, object: string): Promise<CapabilityMapping> {
return (await this.findByIfExists({
type: EngineConstants.spec.capabilityMapping,
[EngineConstants.prop.capability]: capabilityId,
[EngineConstants.prop.object]: InversePath({
subPath: ZeroOrMorePath({ subPath: RDFS.subClassOf as string }),
value: object
})
})) as CapabilityMapping;
}
private async performCapabilityMappingWithArgs(
args: JSONValue,
mapping: Mapping,
capabilityConfig?: CapabilityConfig
): Promise<string | undefined> {
if (mapping[EngineConstants.prop.operationId]) {
return getValueIfDefined<string>(mapping[EngineConstants.prop.operationId])!;
}
if (mapping[EngineConstants.prop.operationId]) {
return getValueIfDefined<string>(mapping[EngineConstants.prop.operationId])!;
}
const capabilityInfoJsonLd = await this.performMapping(
args,
mapping[EngineConstants.prop.operationMapping] as NodeObject,
undefined,
capabilityConfig
);
return getValueIfDefined<string>(capabilityInfoJsonLd[EngineConstants.prop.operationId])!;
}
private async assertCapabilityParamsMatchParameterSchemas(
capabilityParams: any,
capability: Capability
): Promise<void> {
let parametersSchemaObject = capability[EngineConstants.prop.inputs];
if (parametersSchemaObject?.[PROP_ENTITY_ID] && Object.keys(parametersSchemaObject).length === 1) {
parametersSchemaObject = await this.findBy({ id: parametersSchemaObject[PROP_ENTITY_ID] });
}
if (capabilityParams && parametersSchemaObject) {
const capabilityParamsAsJsonLd = {
'@context': getValueIfDefined<ContextDefinition>(capability[EngineConstants.prop.inputsContext]),
[PROP_ENTITY_TYPE]: EngineConstants.spec.inputs,
...capabilityParams
};
const report = await this.convertToQuadsAndValidateAgainstShape(capabilityParamsAsJsonLd, parametersSchemaObject);
if (!report.conforms) {
this.throwValidationReportError(
report,
`${getValueIfDefined(capability[EngineConstants.prop.label])} parameters do not conform to the schema`
);
}
}
}
private async performOpenapiOperationWithCredentials(
operationId: string,
operationA