@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
52 lines • 2.06 kB
JavaScript
import { normalizeOptionalString } from '../../utils/value.js';
import { ATLAS_AI_RESOLVE_STOP_WORDS, ATLAS_AI_RESOLVE_TOKEN_ALIASES } from './resolveLexicon.js';
const normalizeResolveText = value => {
const normalizedValue = normalizeOptionalString(value);
if (!normalizedValue) {
return '';
}
return normalizedValue.normalize('NFKD').replace(/([a-z\d])([A-Z])/g, '$1 $2').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ' ').trim();
};
const addResolveTokenVariant = (tokenPool, token) => {
tokenPool.add(token);
if (token.endsWith('s') && !token.endsWith('ss') && token.length > 4) {
tokenPool.add(token.slice(0, -1));
}
};
export const createResolveTokens = (query, scope) => {
const tokenPool = new Set();
const normalizedInput = normalizeResolveText([normalizeOptionalString(query), normalizeOptionalString(scope)].filter(Boolean).join(' '));
for (const token of normalizedInput.split(/\s+/u).filter(Boolean)) {
if (token.length < 2 || ATLAS_AI_RESOLVE_STOP_WORDS.has(token)) {
continue;
}
addResolveTokenVariant(tokenPool, token);
for (const alias of ATLAS_AI_RESOLVE_TOKEN_ALIASES[token] ?? []) {
tokenPool.add(alias);
}
}
return [...tokenPool.values()].sort((left, right) => left.localeCompare(right));
};
const createResolveFragmentTokenSet = fragments => {
const fragmentTokenSet = new Set();
for (const fragment of Array.isArray(fragments) ? fragments : [fragments]) {
const normalizedFragment = normalizeResolveText(fragment);
if (!normalizedFragment) {
continue;
}
for (const token of normalizedFragment.split(/\s+/u).filter(Boolean)) {
if (token.length < 2) {
continue;
}
addResolveTokenVariant(fragmentTokenSet, token);
}
}
return fragmentTokenSet;
};
export const collectResolveMatchedTokens = (queryTokens, fragments) => {
const fragmentTokenSet = createResolveFragmentTokenSet(fragments);
if (fragmentTokenSet.size === 0) {
return [];
}
return queryTokens.filter(queryToken => fragmentTokenSet.has(queryToken));
};