asksuite-core/AsksuiteProcessingNatural.js

/* eslint-disable no-case-declarations */
const AWSLambdaCaller = require('./services/AWSLambdaCaller');
const Util = require('./util');
const DialogFlowAccessor = require('./services/DialogFlowAccessor');
const KeyMatcherAccessor = require('./services/KeyMatcherAccessor');
const NLPClassifierAccessor = require('./services/NLPClassifierAccessor');
const ExactMatcherAccessor = require('./services/ExactMatcherAccessor');
const OpenAiAcessor = require('./services/OpenAiAcessor');
const defaultFallbackIntent = require('./util/defaultFallbackIntent');
const RequestPreprocessor = require('./util/RequestPreprocessor');
const AsksuiteProcessingNaturalUtils = require('./AsksuiteProcessingNaturalUtils');
const IntEncoder = require('./util/IntEncoder')();
const NodeCache = require('node-cache');
const AsksuiteTranslator = require('./util/Translator');
const StringUtils = require('./util/StringUtils');
const _ = require('lodash');
const PrometheusAcessor = require('./services/PrometheusAccessor');
const SophiaAcessor = require('./services/SophiaAccessor');
const GlobalCacheManager = require('./services/GlobalCacheManager');
const QuoteDataAccessor = require('./services/QuoteDataAccessor');
const MediaResolver = require('./util/MediaResolver');

const DEFAULT_LANGUAGE = 'pt-br';

const GROUP_ID = 'GROUP_ID';
const DIALOG_FLOW = 'DIALOG_FLOW';
const KEY_MATCHER = 'KEY_MATCHER';
const EXACT_MATCHER = 'EXACT_MATCHER';
const OPEN_AI = 'OPEN_AI';
const NLP_CLASSIFIER = 'NLP_CLASSIFIER';
const PROMETHEUS = 'PROMETHEUS';
const SOPHIA = 'SOPHIA';

const intentsNotFoundCache = new NodeCache({ stdTTL: 300, checkperiod: 120 });

const globalCachebleLayers = [{ name: NLP_CLASSIFIER, ttl: 7200 }];

/**
 * @type {GlobalCacheManager}
 */
let globalCacheManager;

class AsksuiteProcessingNatural {
  static get ORDERED_ACCESSORS() {
    return [GROUP_ID, PROMETHEUS, EXACT_MATCHER, NLP_CLASSIFIER, DIALOG_FLOW, KEY_MATCHER, OPEN_AI];
  }

  constructor(config) {
    this.config = config;
    this.utils = new AsksuiteProcessingNaturalUtils(this.config);

    this.exactMatcherAccessor = new ExactMatcherAccessor({
      config: config.config,
      redis: config.redisCore || config.redis,
    });

    this.setGlobalCache();
  }

  resolverOrder(config) {
    const resolvers = AsksuiteProcessingNatural.ORDERED_ACCESSORS;

    if (config.SOPHIA_NLP_ENABLED) {
      return [GROUP_ID, SOPHIA];
    }

    if (!config.RESOLVERS_TO_BE_USED || !config.RESOLVERS_TO_BE_USED.length) {
      return resolvers;
    }

    return resolvers.filter((r) => config.RESOLVERS_TO_BE_USED.includes(r));
  }

  findIntent(originalRequest) {
    return new Promise((resolve, reject) => {
      const executor = async (originalRequest) => {
        let intent = {
          fromCache: false,
        };

        // Formata texto
        const request = await RequestPreprocessor.process(originalRequest);

        const language = this.utils.resolveLanguage(request.language, request.languages);
        const key = `${request.companyId}.${request.userId}.${language}`;

        const {
          keyMatcherAccessor,
          openAIAccessor,
          nlpClassifierAccessor,
          prometheusAccessor,
          sophiaAccessor,
          dialogFlowAccessor,
        } = this.resolveConnectors(request, originalRequest);

        this.dialogFlowAccessor = dialogFlowAccessor;

        const orderOption = this.resolverOrder(request.config);

        request.findIntent = defaultFallbackIntent(request.companyId);
        request.translated = false;
        request.translation = null;

        const processingSteps = [];

        for (const accessorName of orderOption) {
          const cachedIntent = await this.processCacheGlobal(
            accessorName,
            originalRequest,
          );

          if (cachedIntent) {
            intent = cachedIntent;
          } else {
            switch (accessorName) {
              case GROUP_ID:
                const extractedValue = IntEncoder.extract(originalRequest.text);
                if (extractedValue) {
                  intent = this.utils.findDialogByIntent(request, 'continuar_atendimento');
                  const [groupId, attendanceId] = extractedValue.split('-');
                  intent.groupId = groupId;
                  intent.attendanceId = attendanceId;
                }
                break;
              case DIALOG_FLOW:
                intent = await this.resolveDialogFlowAccessor(request, language);
                break;
              case KEY_MATCHER:
                intent = await keyMatcherAccessor.resolveText(request);
                if (Util.isNaoentendi(intent.intent) && language !== DEFAULT_LANGUAGE) {
                  const toSend = await this.utils.copyRequestToDefaultLanguage(request);
                  intent = await keyMatcherAccessor.resolveText(toSend);
                }
                intent.keywordTry = { try: true };
                break;
              case PROMETHEUS:
                const [prometheusResult, nlpClassifierResult] = await Promise.all([
                  prometheusAccessor.resolveText(request, originalRequest, true),
                  nlpClassifierAccessor.resolveText(originalRequest),
                ]);

                intent = prometheusResult;
                intent.processingStep.nlpClassifierResult = nlpClassifierResult;

                if (
                  Util.shouldDisambiguatePrometheusAndNlpResults(
                    prometheusResult,
                    nlpClassifierResult,
                  )
                ) {
                  intent.processingStep.disambiguate = true;
                  intent.dialogsToDisambiguate = [
                    {
                      intent: intent.intent,
                      dialog: intent.dialog,
                      intentLabel: intent?.intentLabel,
                    },
                    { intent: nlpClassifierResult.intent, dialog: nlpClassifierResult.dialog },
                  ];
                }
                break;
              case SOPHIA:
                if (request.config.SOPHIA_NLP_ENABLED && !intent?.groupId) {
                  intent = await sophiaAccessor.resolveText(originalRequest);
                }
                break;
              case EXACT_MATCHER:
                if (request.defaultLanguage) {
                  intent = await this.exactMatcherAccessor.resolveText(request);
                }
                break;
              case OPEN_AI:
                intent = await openAIAccessor.resolveText(originalRequest);
                break;
              case NLP_CLASSIFIER:
                if (nlpClassifierAccessor.isSupportedLanguage(language)) {
                  const [nlpClassifierResult, dialogFlowResult] = await Promise.all([
                    nlpClassifierAccessor.resolveText(originalRequest),
                    this.resolveDialogFlowAccessor(request, language),
                  ]);
                  intent = nlpClassifierResult;
                  intent.processingStep.dialogFlowResult = dialogFlowResult;
                  if (Util.shouldDisambiguateNlpAndDialogFlowResults(intent, dialogFlowResult)) {
                    intent.processingStep.disambiguate = true;
                    intent.dialogsToDisambiguate = [
                      { intent: intent.intent, dialog: intent.dialog },
                      { intent: dialogFlowResult.intent, dialog: dialogFlowResult.dialog },
                    ];
                  }
                }
                break;
            }
          }

          this.getProcessorStep(intent, accessorName, request.language, processingSteps);
          if (request.findIntent && request.findIntent.groupId) {
            intent.groupId = request.findIntent.groupId;
          }
          request.findIntent = intent;

          if (intent.intent) {
            if (this.shouldUseGlobalCache(accessorName)) {
              try {
                await globalCacheManager.set(accessorName, originalRequest, intent);
              } catch (err) {
                console.log(`Error setting ${intent} intent into ${accessorName}`, err);
              }
            }
          }

          if (accessorName === SOPHIA || (intent.intent && !Util.isNaoentendi(intent.intent))) {
            intent.resolver = accessorName;
            if (intent.resolver === KEY_MATCHER) {
              intent.keywordTry.match = intent.intent;
              intent.keywordTry.intent = intent.intent;
            }
            // FOR exit condition
            break;
          }
        }

        if (Util.isNaoentendi(intent.intent)) {
          let count =
            originalRequest.subsequentNotUnderstoodCount || intentsNotFoundCache.get(key) || 0;
          intentsNotFoundCache.set(key, ++count);

          const { ATTEMPTS_BEFORE_DETECTING_LANGUAGE, LANGUAGE_DETECTION_CONFIDENCE } =
            request.config;

          if (count >= ATTEMPTS_BEFORE_DETECTING_LANGUAGE) {
            const detection = await this.utils.detectLanguage(request);

            if (detection.confidence >= LANGUAGE_DETECTION_CONFIDENCE) {
              const languageCode = detection.isSupported
                ? detection.language.code
                : detection.fallbackLanguage.code;

              if (request.language.substring(0, 2) !== languageCode.substring(0, 2)) {
                intent.languageDetected = detection;
                intentsNotFoundCache.del(key);
              }
            }
          }

          if (count > 1 && !intent.languageDetected && !intent.groupId) {
            const fallbackIntent = defaultFallbackIntent(request.companyId, count);
            intent = this.utils.findDialogByIntent(request, fallbackIntent.intent);
          }
        } else {
          intentsNotFoundCache.del(key);
        }

        intent.translated = request.translated;
        intent.translation = request.translation;
        intent.processingSteps = processingSteps;

        return intent;
      };

      executor(originalRequest).then(resolve).catch(reject);
    });
  }

  async processCacheGlobal(accessorName, originalRequest) {
    if (this.shouldUseGlobalCache(accessorName)) {
      try {
        const cachedIntent = await globalCacheManager.get(accessorName, originalRequest);

        if (cachedIntent) {
          console.log(`OBTAINED ${cachedIntent.intent} BY CACHE FOR LAYER ${accessorName}`);
          return {
            ...cachedIntent,
            processingStep: { fromGlobalCache: true },
          };
        }
      } catch (e) {
        console.error(`ERROR ON GLOBAL CACHE`, e);
      }
    }

    return null;
  }

  resolveConnectors(request, originalRequest) {
    const awsLambdaCaller = new AWSLambdaCaller(request.config.CORE_LAMBDA_AWS);
    const dialogFlowAccessor = new DialogFlowAccessor(awsLambdaCaller);

    const keyMatcherAccessor = new KeyMatcherAccessor(
      awsLambdaCaller,
      request.config.OPEN_AI_ENABLED,
    );

    const openAIAccessor = new OpenAiAcessor(
      request.config.OPEN_AI_URL,
      request.config.OPEN_AI_ENABLED,
      originalRequest,
    );

    const nlpClassifierAccessor = new NLPClassifierAccessor(
      request.config.NLP_CLASSIFIER_LAYER_URL,
      request.config.NLP_CLASSIFIER_PARAMETERS,
    );

    const prometheusAccessor = new PrometheusAcessor(
      request.config.PROMETHEUS_PARAMETERS,
      this.config.redis,
    );

    const sophiaAccessor = new SophiaAcessor(
      request.config.SOPHIA_NLP_URL,
      request.config.SOPHIA_NLP_ENABLED,
      originalRequest,
    );

    return {
      keyMatcherAccessor,
      openAIAccessor,
      nlpClassifierAccessor,
      prometheusAccessor,
      sophiaAccessor,
      dialogFlowAccessor,
    };
  }

  async resolveDialogFlowAccessor(request, language) {
    if (!this.dialogFlowResult) {
      request = _.cloneDeep(request);

      this.dialogFlowResult = await this.utils.findIntentDialogFlow(
        this.dialogFlowAccessor,
        request,
        false,
      );

      this.dialogFlowResult.processingStep = this.dialogFlowResult.processingStep || {};

      if (Util.isNaoentendi(this.dialogFlowResult.intent) && language !== DEFAULT_LANGUAGE) {
        this.dialogFlowResult = await this.utils.findIntentDialogFlow(
          this.dialogFlowAccessor,
          request,
          true,
        );
      }

      const processingStep = this.dialogFlowResult.processingStep;
      const isChangeLanguage =
        Util.isChangeLanguage(this.dialogFlowResult.intent) ||
        Util.isChangeLanguage(processingStep?.intentFound);

      processingStep.isChangeLanguageRequest = isChangeLanguage;
      if (
        isChangeLanguage &&
        this.dialogFlowResult.parameterFields &&
        this.dialogFlowResult.parameterFields.language
      ) {
        await this.processChangeLanguageRequest(request);
      }
    }
    return this.dialogFlowResult;
  }

  getProcessorStep(intent, resolver, language, processingSteps) {
    if (intent && intent.processingStep) {
      try {
        processingSteps.push({
          resolver,
          language,
          processingStep: JSON.parse(
            JSON.stringify(intent.processingStep, this.removeCircularReferencesOnStringify()),
          ),
        });

        delete intent.processingStep;
      } catch (e) {
        console.log('[AsksuiteProcessingNatural.getProcessorStep] error', e);
        console.log(
          '[AsksuiteProcessingNatural.getProcessorStep] error intent',
          JSON.stringify(intent),
        );
      }
    }
  }

  removeCircularReferencesOnStringify() {
    const seen = new WeakSet();
    return (_, value) => {
      if (typeof value === 'object' && value !== null) {
        if (seen.has(value)) {
          return undefined;
        }
        seen.add(value);
      }
      return value;
    };
  }

  async processChangeLanguageRequest(request) {
    if (
      !(typeof this.dialogFlowResult.parameterFields === 'object') ||
      !this.dialogFlowResult.parameterFields.language
    ) {
      return;
    }

    this.dialogFlowResult.intent = 'trocar_idioma';

    const languageListValue = this.dialogFlowResult.parameterFields.language.listValue || {
      values: [],
    };

    const googleTranslateKey = request.config.GOOGLE_TRANSLATE_KEY;

    const currentLanguageCode = request.language.substring(0, 2);

    let languageRequested;

    // targetLanguage is used to discover which language the traveller is using
    let targetLanguage = request.translated ? 'pt' : undefined;

    if (!targetLanguage) {
      const detectedLanguage = await AsksuiteTranslator.detectLanguage(
        googleTranslateKey,
        request.text,
      );

      if (detectedLanguage && detectedLanguage.length) {
        targetLanguage = detectedLanguage[0].language;
      } else {
        targetLanguage = currentLanguageCode;
      }
    }

    for (const value of languageListValue.values) {
      const language = StringUtils.unaccent(value.stringValue);

      const valueLanguageInfo = await AsksuiteTranslator.getCodeByLanguage(
        googleTranslateKey,
        language,
        targetLanguage,
      );

      // eslint-disable-next-line eqeqeq
      if (valueLanguageInfo && valueLanguageInfo.code != currentLanguageCode) {
        valueLanguageInfo.name = valueLanguageInfo.name.replace(/\(\w+\)/g, '');
        languageRequested = valueLanguageInfo;
        break;
      }
    }

    if (!languageRequested) {
      return;
    }

    const languageInfo = await AsksuiteTranslator.getLanguageByCode(
      googleTranslateKey,
      languageRequested.code,
      languageRequested.code,
    );

    if (!languageInfo) {
      return;
    }

    const enIsSupported = this.utils.isLanguageSupportedByCompany(request, 'en');
    const fallbackLanguageCode = enIsSupported ? 'en' : currentLanguageCode;

    const fallbackLanguage = await AsksuiteTranslator.getLanguageByCode(
      googleTranslateKey,
      fallbackLanguageCode,
      fallbackLanguageCode,
    );
    const isSupported = this.utils.isLanguageSupportedByCompany(request, languageInfo.code);

    if (!isSupported) {
      languageInfo.name = (
        await AsksuiteTranslator.getLanguageByCode(
          googleTranslateKey,
          languageInfo.code,
          fallbackLanguageCode,
        )
      ).name;
    }

    const languageDetected = {
      language: languageInfo,
      isSupported,
      fallbackLanguage,
    };

    this.dialogFlowResult.languageDetected = languageDetected;
    this.dialogFlowResult.processingStep.languageDetected = languageDetected;
  }

  resolveMedia(request) {
    return new Promise((resolve, reject) => {
      try {
        const mediaResolver = new MediaResolver(request.companyId);
        mediaResolver.resolve(request.media, request.intents).then(resolve).catch(reject);
      } catch (e) {
        reject(e);
      }
    });
  }

  async extractQuoteData(request) {
    request = await RequestPreprocessor.process(request);

    const awsLambdaCaller = new AWSLambdaCaller(request.config.CORE_LAMBDA_AWS);
    const quoteDataAccessor = new QuoteDataAccessor(awsLambdaCaller);

    return await quoteDataAccessor.resolveText(request);
  }

  async extractTextNumbers(request) {
    request = await RequestPreprocessor.process(request);

    const awsLambdaCaller = new AWSLambdaCaller(request.config.CORE_LAMBDA_AWS);
    const quoteDataAccessor = new QuoteDataAccessor(awsLambdaCaller);

    return await quoteDataAccessor.resolveTextNumbers(request);
  }

  setGlobalCache() {
    if (globalCacheManager) {
      return globalCacheManager;
    }

    if (!this.config.redis) {
      return;
    }

    globalCacheManager = new GlobalCacheManager(this.config.redis);
    globalCacheManager.addLayers(globalCachebleLayers);
  }

  shouldUseGlobalCache(accessorName) {
    return globalCacheManager && globalCacheManager.isGlobalLayer(accessorName);
  }
}

module.exports = AsksuiteProcessingNatural;