UNPKG

osls

Version:

Open-source alternative to Serverless Framework

690 lines (661 loc) 28.2 kB
// Having variables meta, configuration, and sources setup, attempt to resolve all variables 'use strict'; const ensureArray = require('type/array/ensure'); const ensureSet = require('type/set/ensure'); const ensureMap = require('type/map/ensure'); const ensureString = require('type/string/ensure'); const coerceString = require('type/string/coerce'); const isError = require('type/error/is'); const isObject = require('type/object/is'); const isPlainObject = require('type/plain-object/is'); const ensurePlainObject = require('type/plain-object/ensure'); const ensurePlainFunction = require('type/plain-function/ensure'); const memoizeMethods = require('memoizee/methods'); const path = require('path'); const util = require('util'); const d = require('d'); const ServerlessError = require('../../serverless-error'); const humanizePropertyPath = require('./humanize-property-path-keys'); const parse = require('./parse'); const { parseEntries } = require('./resolve-meta'); const VariableSourceResolutionError = require('./source-resolution-error'); const objPropertyIsEnumerable = Object.prototype.propertyIsEnumerable; const variableProcessingErrorNames = new Set(['ServerlessError', 'VariableSourceResolutionError']); let lastResolutionBatchId = 0; const resolveSourceValuesVariables = (sourceValues) => { // Value is a result of concatenation of string values coming from multiple sources. // Parse variables in each string part individually - it's to avoid accidental // resolution of variable-like notation which may surface after joining two values. // Also that way we do not accidentally re-parse an escaped variables notation let baseIndex = 0; let resolvedValueVariables = null; for (const sourceValue of sourceValues) { const sourceValueVariables = parse(sourceValue); if (sourceValueVariables) { for (const sourceValueVariable of sourceValueVariables) { if (sourceValueVariable.end) { sourceValueVariable.start += baseIndex; sourceValueVariable.end += baseIndex; } else { sourceValueVariable.start = baseIndex; sourceValueVariable.end = baseIndex + sourceValue.length; } } if (!resolvedValueVariables) resolvedValueVariables = []; resolvedValueVariables.push(...sourceValueVariables); } baseIndex += sourceValue.length; } return resolvedValueVariables; }; class VariablesResolver { constructor({ serviceDir, configuration, variablesMeta, sources, options, fulfilledSources, propertyPathsToResolve, variableSourcesInConfig, }) { this.serviceDir = serviceDir; this.configuration = configuration; this.variablesMeta = variablesMeta; this.sources = sources; this.options = options; this.fulfilledSources = fulfilledSources; this.propertyDependenciesMap = new Map(); this.propertyResolutionNestDepthMap = new Map(); // It is used to record all encountered variable sources in configuration for telemetry purposes this.variableSourcesInConfig = variableSourcesInConfig || new Set(); // Resolve all variables simultaneously // Resolution batches are identified to ensure source resolution cache is not shared among them. // (each new resolution batch has access to extended configuration structure // and cached source resolver may block access to already resolved structure) const resolutionBatchId = ++lastResolutionBatchId; if (propertyPathsToResolve) { return Promise.all( Array.from(propertyPathsToResolve, (propertyPathToResolve) => Promise.all( Array.from(variablesMeta.keys(), (propertyPath) => { if ( propertyPathToResolve.startsWith(`${propertyPath}\0`) || propertyPath === propertyPathToResolve || propertyPath.startsWith(`${propertyPathToResolve}\0`) ) { return this.resolveProperty(resolutionBatchId, propertyPath); } return null; }) ) ) ).then(() => {}); } return Promise.all( Array.from(variablesMeta.keys(), (propertyPath) => this.resolveProperty(resolutionBatchId, propertyPath) ) ).then(() => {}); } async resolveVariables(resolutionBatchId, propertyPath, valueMeta) { // Resolve all variables configured in given string value. await Promise.all( valueMeta.variables.map(async (variableMeta) => { if (!variableMeta.sources) { // Variable was already resolved in previous resolution phase return; } await this.resolveVariable(resolutionBatchId, propertyPath, variableMeta); if (!variableMeta.error) { // Variable successfully resolved return; } if (valueMeta.error) return; delete valueMeta.variables; valueMeta.error = variableMeta.error; }) ); if (!valueMeta.variables) { // Abort, as either: // - Resolution of some variables errored // - Value was already resolved in other resolution batch // (triggered by depending property resolver) return; } if (valueMeta.variables.some((variableMeta) => variableMeta.sources)) { // if some of the variables could not be resolved at this point, abort return; } // All variables for value resolved, rebuilt value if (valueMeta.variables.length === 1 && !valueMeta.variables[0].end) { // String value is represented from start to end by single variable. // Replace with resolved value as-is valueMeta.value = valueMeta.variables[0].value; delete valueMeta.variables; return; } // String value is either constructed from more than one variable // or is partially constructed with a variable. // In such case, end value is always a string, reconstructed below const sourceValues = []; const rawValue = valueMeta.value; let lastIndex = 0; for (const { start, end, value } of valueMeta.variables) { const stringValue = coerceString(value); if (stringValue == null) { delete valueMeta.variables; valueMeta.error = new ServerlessError( `Cannot resolve variable at "${humanizePropertyPath( propertyPath.split('\0') )}": String value consist of variable which resolve with non-string value`, 'NON_STRING_VARIABLE_RESULT' ); return; } sourceValues.push(rawValue.slice(lastIndex, start), stringValue); lastIndex = end; } sourceValues.push(rawValue.slice(lastIndex)); valueMeta.value = sourceValues.join(''); valueMeta.sourceValues = sourceValues; delete valueMeta.variables; } async resolveVariable(resolutionBatchId, propertyPath, variableMeta) { // Resolve a single variable, which could be configured with multiple // (first-choice, and fallback) sources // Work on a copy, as with mulitple resolution batches, there's a rare possibility of same // variable being resolved multiple times at once const sources = Array.from(variableMeta.sources); for (const source of sources) { if (source.type) this.variableSourcesInConfig.add(source.type); } let sourceData = sources[0]; do { const sourceMeta = this.sources[sourceData.type]; if (!sourceMeta) { // Unknown source, skip resolution (it'll leave variable as not resolved) return; } let resolvedData; try { resolvedData = await this.resolveVariableSource( resolutionBatchId, propertyPath, sourceData ); } catch (error) { /* istanbul ignore next */ if ( !error || !error.constructor || !variableProcessingErrorNames.has(error.constructor.name) ) { // Programmer error (ideally dead path) throw error; } if (error.code === 'MISSING_VARIABLE_DEPENDENCY') { // Resolution internally depends on unknown source, silently abort // (it'll leave variable as not resolved) variableMeta.sources = sources; return; } // Resolution error, which signals configuration error // Mark as not recoverable error and abort. delete variableMeta.sources; variableMeta.error = error; return; } if (resolvedData.value != null) { // Source successfully resolved. Accept as final value delete variableMeta.sources; variableMeta.value = resolvedData.value; return; } if (resolvedData.isPending) { // Source resolved with "null", but is marked as not yet fulfilled // (not having all data available at this point) // Silently abort (it'll leave variable as not resolved) variableMeta.sources = sources; return; } sources.shift(); const previousSourceData = sourceData; sourceData = sources[0]; if (!sourceData) { // Last source reported no value and there's no further fallback, we treat it as an error // In further processing ideally it should be surfaced and prevent command from continuing delete variableMeta.sources; const detailedErrorMessage = resolvedData.eventualErrorMessage || `Value not found at "${previousSourceData.type}" source`; variableMeta.error = new ServerlessError( `Cannot resolve variable at "${humanizePropertyPath( propertyPath.split('\0') )}": ${detailedErrorMessage}`, 'MISSING_VARIABLE_RESULT' ); return; } } while (sourceData.type); // Fallback to static value source delete variableMeta.sources; variableMeta.value = sourceData.value; } async resolveVariableSource(resolutionBatchId, propertyPath, sourceData) { // Resolve variables in source dependencies (params and address) and resolve the source if (sourceData.params) { // Ensure to have all eventual variables in params resolved await Promise.all( sourceData.params.map(async (param) => { if (!param.variables) return; await this.resolveVariables(resolutionBatchId, propertyPath, param); await this.resolveInternalResult(resolutionBatchId, propertyPath, param); }) ); } if (sourceData.address && sourceData.address.variables) { // Ensure to have all eventual variables in address resolved await this.resolveVariables(resolutionBatchId, propertyPath, sourceData.address); await this.resolveInternalResult(resolutionBatchId, propertyPath, sourceData.address); } return JSON.parse( JSON.stringify( await (async () => { try { return await this.resolveSource(resolutionBatchId, propertyPath, sourceData); } catch (error) { if (isError(error)) { if (error.code === 'MISSING_VARIABLE_DEPENDENCY') throw error; if ( error.constructor.name === 'ServerlessError' && error.message.startsWith('Cannot resolve variable at ') ) { throw error; } } let isServerlessError = false; const originalErrorMessage = (() => { if (!isError(error)) { return `Non-error exception: ${util.inspect(error)}`; } else if (error.constructor.name === 'ServerlessError') { isServerlessError = true; return error.message; } else if (error.constructor.name === 'VariableSourceResolutionError') { return error.message; } return error.stack; })(); throw new (isServerlessError ? ServerlessError : VariableSourceResolutionError)( `Cannot resolve variable at "${humanizePropertyPath( propertyPath.split('\0') )}": ${originalErrorMessage}`, 'VARIABLE_RESOLUTION_ERROR' ); } })() ) ); } async resolveInternalResult(resolutionBatchId, propertyPath, valueMeta, nestTracker = 10) { if (valueMeta.error) throw valueMeta.error; if (valueMeta.variables) { throw new ServerlessError('Cannot resolve variable', 'MISSING_VARIABLE_DEPENDENCY'); } if (!nestTracker) { throw new ServerlessError( `Cannot resolve variable at "${humanizePropertyPath( propertyPath.split('\0') )}": Excessive variables nest depth`, 'EXCESSIVE_RESOLVED_VARIABLES_NEST_DEPTH' ); } if (typeof valueMeta.value === 'string') { const valueVariables = (() => { try { if (valueMeta.sourceValues) return resolveSourceValuesVariables(valueMeta.sourceValues); return parse(valueMeta.value); } catch (error) { error.message = `Cannot resolve variable at "${humanizePropertyPath( propertyPath.split('\0') )}": Approached variable syntax error in resolved value "${valueMeta.value}": ${ error.message }`; delete valueMeta.value; valueMeta.error = error; throw error; } })(); if (!valueVariables) return; valueMeta.variables = valueVariables; delete valueMeta.sourceValues; await this.resolveVariables(resolutionBatchId, propertyPath, valueMeta); await this.resolveInternalResult(resolutionBatchId, propertyPath, valueMeta, nestTracker - 1); return; } const valueEntries = (() => { if (isPlainObject(valueMeta.value)) return Object.entries(valueMeta.value); return Array.isArray(valueMeta.value) ? valueMeta.value.entries() : null; })(); if (!valueEntries) return; const propertyVariablesMeta = parseEntries(valueEntries, [], new Map()); for (const [propertyKeyPath, propertyValueMeta] of propertyVariablesMeta) { await this.resolveVariables(resolutionBatchId, propertyPath, propertyValueMeta); await this.resolveInternalResult( resolutionBatchId, propertyPath, propertyValueMeta, nestTracker - 1 ); const propertyKeyPathKeys = propertyKeyPath.split('\0'); const targetKey = propertyKeyPathKeys[propertyKeyPathKeys.length - 1]; let targetObject = valueMeta.value; for (const parentKey of propertyKeyPathKeys.slice(0, -1)) { targetObject = targetObject[parentKey]; } targetObject[targetKey] = propertyValueMeta.value; } } validateCrossPropertyDependency(dependentPropertyPath, dependencyPropertyPath) { if (dependentPropertyPath === dependencyPropertyPath) { throw new ServerlessError( `Circular reference. "${humanizePropertyPath( dependentPropertyPath.split('\0') )}" refers to itself`, 'CIRCULAR_PROPERTY_DEPENDENCY' ); } const dependencyDependencies = this.propertyDependenciesMap.get(dependencyPropertyPath); if (!dependencyDependencies) return; if (dependencyDependencies.has(dependentPropertyPath)) { throw new ServerlessError( `Circular reference among "${humanizePropertyPath( dependentPropertyPath.split('\0') )}" and "${humanizePropertyPath(dependencyPropertyPath.split('\0'))}" properties`, 'CIRCULAR_PROPERTY_DEPENDENCY' ); } for (const dependencyDependency of dependencyDependencies) { this.validateCrossPropertyDependency(dependentPropertyPath, dependencyDependency); } } registerCrossPropertyDependency(dependentPropertyPath, dependencyPropertyPath) { this.validateCrossPropertyDependency(dependentPropertyPath, dependencyPropertyPath); if (!this.propertyDependenciesMap.has(dependentPropertyPath)) { this.propertyDependenciesMap.set(dependentPropertyPath, new Set()); } this.propertyDependenciesMap.get(dependentPropertyPath).add(dependencyPropertyPath); } async resolveDependentProperty( resolutionBatchId, dependentPropertyPath, dependencyPropertyPathKeys ) { let value = this.configuration; for (const [index, key] of dependencyPropertyPathKeys.entries()) { if (value == null) { value = undefined; break; } const depPropertyPath = dependencyPropertyPathKeys.slice(0, index + 1).join('\0'); if (this.variablesMeta.has(depPropertyPath)) { this.registerCrossPropertyDependency(dependentPropertyPath, depPropertyPath); await this.resolveProperty(resolutionBatchId, depPropertyPath); } const depValueMeta = this.variablesMeta.get(depPropertyPath); if (depValueMeta) { if (depValueMeta.error) throw depValueMeta.error; throw new ServerlessError('Cannot resolve variable', 'MISSING_VARIABLE_DEPENDENCY'); } value = value[key]; } if (!isObject(value)) return value; const depPropertyNestPath = dependencyPropertyPathKeys.length ? `${dependencyPropertyPathKeys.join('\0')}\0` : ''; await Promise.all( Array.from(this.variablesMeta.keys()).map(async (propertyPath) => { if (!propertyPath.startsWith(depPropertyNestPath)) return; this.registerCrossPropertyDependency(dependentPropertyPath, propertyPath); await this.resolveProperty(resolutionBatchId, propertyPath); const valueMeta = this.variablesMeta.get(propertyPath); if (!valueMeta) return; if (valueMeta.error) throw valueMeta.error; throw new ServerlessError('Cannot resolve variable', 'MISSING_VARIABLE_DEPENDENCY'); }) ); return value; } } Object.defineProperties( VariablesResolver.prototype, memoizeMethods({ resolveProperty: d( async function self(resolutionBatchId, propertyPath) { const valueMeta = this.variablesMeta.get(propertyPath); if (!valueMeta.variables) { // Lack of `.variables` means that there was an attempt to resolve property in previous pass // but it errored. // In normal flow, we will not re-attempt variables resolution in such case (so that's a dead path) // but for algorithm completeness (and for testing convenience) such scenario is handled here return; } await this.resolveVariables(resolutionBatchId, propertyPath, valueMeta); if (valueMeta.variables || valueMeta.error) { // Having `.variables` still here, means we could not attempt to resolve the variable // (e.g.source resolution methods were not provided, or source is seen as not yet // fulfilled and responded with no value for given params/address). // In such case resolution can be retried in later turn, after missing data is provided. // // Having `.error` means, resolution errored (with no recovery plan for it) // Such error ideally should be exposed with command crash in resolution consumer logic return; } // Variable(s) for a property where successfully resolved, still it can be an object // (or an array) containing a values with variables in it. const propertyPathKeys = propertyPath.split('\0'); const { value, sourceValues } = valueMeta; let propertyVariablesMeta; if (typeof value === 'string') { const valueVariables = (() => { try { if (sourceValues) return resolveSourceValuesVariables(sourceValues); return parse(value); } catch (error) { error.message = `Cannot resolve variable at "${humanizePropertyPath( propertyPathKeys )}": Approached variable syntax error in resolved value "${value}": ${error.message}`; delete valueMeta.value; valueMeta.error = error; return null; } })(); if (valueVariables) { propertyVariablesMeta = new Map([[propertyPath, { value, variables: valueVariables }]]); } else if (valueMeta.error) { return; } } else { const valueEntries = (() => { if (isPlainObject(value)) return Object.entries(value); return Array.isArray(value) ? value.entries() : null; })(); if (valueEntries) { propertyVariablesMeta = parseEntries(valueEntries, propertyPathKeys, new Map()); } } if (propertyVariablesMeta && propertyVariablesMeta.size) { // Register variables found in resolved value const nestDepth = (this.propertyResolutionNestDepthMap.get(propertyPath) || 0) + 1; if (nestDepth > 10) { // Found a deep recursion where value that hosts variables, resolves // with value that hosts variables etc. // It's a likely signal of circular value resolution error. Abort early delete valueMeta.value; valueMeta.error = new ServerlessError( `Cannot resolve variable at "${humanizePropertyPath( propertyPathKeys )}": Excessive property variables nest depth`, 'EXCESSIVE_RESOLVED_PROPERTIES_NEST_DEPTH' ); return; } for (const [subPropertyKeyPath, subPropertyValue] of propertyVariablesMeta) { this.propertyResolutionNestDepthMap.set(subPropertyKeyPath, nestDepth); this.variablesMeta.set(subPropertyKeyPath, subPropertyValue); } } // Assign resolved value to configuration, and remove variable from variables meta registry if (!propertyVariablesMeta || !propertyVariablesMeta.has(propertyPath)) { this.variablesMeta.delete(propertyPath); } const targetKey = propertyPathKeys[propertyPathKeys.length - 1]; let targetObject = this.configuration; for (const parentKey of propertyPathKeys.slice(0, -1)) { targetObject = targetObject[parentKey]; } targetObject[targetKey] = value; if (!propertyVariablesMeta || !propertyVariablesMeta.size) return; if (propertyVariablesMeta.has(propertyPath)) { await self.call(this, resolutionBatchId, propertyPath); return; } // Resolve variables found in resolved value const newResolutionBatchId = ++lastResolutionBatchId; await Promise.all( Array.from(propertyVariablesMeta.keys()).map((subPropertyPath) => this.resolveProperty(newResolutionBatchId, subPropertyPath) ) ); }, { primitive: true, /* We're fine with caching rejections here, hence no "promise: true" option */ } ), resolveSource: d( async function (resolutionBatchId, propertyPath, sourceData) { // Resolve value from variables source, and ensure it's a typical JSON value. const result = await this.sources[sourceData.type].resolve({ params: sourceData.params && sourceData.params.map((param) => param.value), address: sourceData.address && sourceData.address.value, options: this.options, isSourceFulfilled: this.fulfilledSources.has(sourceData.type), serviceDir: this.serviceDir, // TODO: Remove `servicePath` with next major servicePath: this.serviceDir, resolveConfigurationProperty: async (dependencyPropertyPathKeys) => this.resolveDependentProperty( resolutionBatchId, propertyPath, ensureArray(dependencyPropertyPathKeys, { ensureItem: ensureString }) ), resolveVariable: async (variableString) => { variableString = `\${${ensureString(variableString, { Error: ServerlessError, name: 'variableString', errorCode: 'INVALID_VARIABLE_INPUT_TYPE', })}}`; const variableData = parse(variableString); if (!variableData) { throw new ServerlessError( `Invalid variable value: "${variableString}"`, 'INVALID_VARIABLE_INPUT' ); } const meta = { value: variableString, variables: variableData, }; await this.resolveVariables(resolutionBatchId, propertyPath, meta); await this.resolveInternalResult(resolutionBatchId, propertyPath, meta); return meta.value; }, resolveVariablesInString: async (stringValue) => { stringValue = ensureString(stringValue, { Error: ServerlessError, name: 'variableString', errorCode: 'INVALID_STRING_INPUT_TYPE', }); const variableData = parse(stringValue); if (!variableData) return stringValue; const meta = { value: stringValue, variables: variableData, }; await this.resolveVariables(resolutionBatchId, propertyPath, meta); await this.resolveInternalResult(resolutionBatchId, propertyPath, meta); return meta.value; }, }); ensurePlainObject(result, { errorMessage: `Unexpected "${sourceData.type}" source result: %v`, Error: VariableSourceResolutionError, errorCode: 'UNEXPECTED_RESULT', }); if (!objPropertyIsEnumerable.call(result, 'value')) { throw new VariableSourceResolutionError( `Unexpected "${sourceData.type}" source result: Missing "value" property`, 'UNEXPECTED_RESULT_VALUE' ); } const resultValue = result.value; let normalizedResultValue; try { normalizedResultValue = JSON.parse(JSON.stringify(resultValue)); } catch (error) { throw new VariableSourceResolutionError( `Source "${sourceData.type}" returned not supported result: "${util.inspect( resultValue )}"`, 'UNSUPPORTED_VARIABLE_RESOLUTION_RESULT' ); } if ( !isPlainObject(resultValue) && !Array.isArray(resultValue) && resultValue !== normalizedResultValue ) { throw new VariableSourceResolutionError( `Source "${sourceData.type}" returned not supported result: "${util.inspect( resultValue )}"`, 'UNSUPPORTED_VARIABLE_RESOLUTION_RESULT' ); } return result; }, { normalizer: ([resolutionBatchId, , { type, params, address }]) => { return [ resolutionBatchId, type, params && params.map((param) => JSON.stringify(param.value)).join(), address == null ? undefined : JSON.stringify(address), ].join('\n'); }, /* We're fine caching rejections here, hence no "promise: true" option */ } ), }) ); module.exports = async (data) => { // Input sanity check // Note: this function is considered private, if there's a crash here, it signals an internal bug data = { ...ensurePlainObject(data) }; data.serviceDir = path.resolve(ensureString(data.serviceDir)); ensurePlainObject(data.configuration); ensureMap(data.variablesMeta); ensurePlainObject(data.sources); for (const { resolve } of Object.values(data.sources)) ensurePlainFunction(resolve); ensurePlainObject(data.options); ensureSet(data.fulfilledSources); ensureSet(data.propertyPathsToResolve, { isOptional: true }); if (data.propertyPathsToResolve) { data.propertyPathsToResolve = new Set(Array.from(data.propertyPathsToResolve, ensureString)); } // Note: Below construct returns a promise, and not an actual VariablesResolver instance // Class construct is used purely for internal convenience // (functions reuse, state management, parallel executions safety) return new VariablesResolver(data); };