langextract
Version:
A TypeScript library for extracting structured and grounded information from text using LLMs
249 lines • 10.3 kB
JavaScript
"use strict";
/**
* Copyright 2025 kmbro.
*
* This is a TypeScript translation of the original Python LangExtract library
* by Google LLC (https://github.com/google/langextract).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.Resolver = exports.ResolverParsingError = void 0;
/**
* Library for resolving LLM output.
*/
const yaml = __importStar(require("js-yaml"));
const types_1 = require("./types");
const tokenizer_1 = require("./tokenizer");
const FUZZY_ALIGNMENT_MIN_THRESHOLD = 0.75;
class ResolverParsingError extends Error {
constructor(message) {
super(message);
this.name = "ResolverParsingError";
}
}
exports.ResolverParsingError = ResolverParsingError;
class Resolver {
constructor(options = {}) {
this._fenceOutput = options.fenceOutput ?? true;
this._constraint = options.constraint ?? { constraintType: "none" };
this._formatType = options.formatType ?? types_1.FormatType.JSON;
this.extractionIndexSuffix = options.extractionIndexSuffix;
this.extractionAttributesSuffix = options.extractionAttributesSuffix ?? "_attributes";
}
get fenceOutput() {
return this._fenceOutput;
}
set fenceOutput(value) {
this._fenceOutput = value;
}
get formatType() {
return this._formatType;
}
set formatType(value) {
this._formatType = value;
}
resolve(inputText, options = {}) {
try {
const extractionData = this.stringToExtractionData(inputText);
return this.extractOrderedExtractions(extractionData);
}
catch (error) {
if (options.suppressParseErrors) {
console.warn("Parse error suppressed:", error);
return [];
}
throw new ResolverParsingError(`Failed to resolve input text: ${error}`);
}
}
align(extractions, sourceText, tokenOffset, charOffset = 0, enableFuzzyAlignment = true, fuzzyAlignmentThreshold = FUZZY_ALIGNMENT_MIN_THRESHOLD) {
const alignedExtractions = [];
const sourceTokens = (0, tokenizer_1.tokenize)(sourceText).tokens;
for (const extraction of extractions) {
const alignedExtraction = this.alignSingleExtraction(extraction, sourceTokens, sourceText, tokenOffset, charOffset, enableFuzzyAlignment, fuzzyAlignmentThreshold);
if (alignedExtraction) {
alignedExtractions.push(alignedExtraction);
}
}
return alignedExtractions;
}
extractAndParseContent(inputString) {
let content = inputString.trim();
// Remove fence markers if present
if (this._fenceOutput) {
const fenceRegex = /```(?:json|yaml|yml)\n?([\s\S]*?)\n?```/;
const match = content.match(fenceRegex);
if (match) {
content = match[1].trim();
}
}
try {
if (this._formatType === types_1.FormatType.JSON) {
return JSON.parse(content);
}
else if (this._formatType === types_1.FormatType.YAML) {
return yaml.load(content);
}
else {
throw new Error(`Unsupported format type: ${this._formatType}`);
}
}
catch (error) {
throw new Error(`Failed to parse content as ${this._formatType}: ${error}`);
}
}
stringToExtractionData(inputString) {
const parsed = this.extractAndParseContent(inputString);
if (Array.isArray(parsed)) {
return parsed;
}
else if (parsed && typeof parsed === "object" && "extractions" in parsed) {
return parsed.extractions;
}
else {
throw new Error("Invalid extraction data format");
}
}
extractOrderedExtractions(extractionData) {
const extractions = [];
for (let i = 0; i < extractionData.length; i++) {
const data = extractionData[i];
for (const [key, value] of Object.entries(data)) {
if (key === this.extractionAttributesSuffix || key.endsWith(this.extractionAttributesSuffix)) {
continue; // Skip attribute fields
}
if (typeof value === "string") {
const extraction = {
extractionClass: key,
extractionText: value,
extractionIndex: i,
};
// Add attributes if available
const attributesKey = `${key}${this.extractionAttributesSuffix}`;
if (data[attributesKey] && typeof data[attributesKey] === "object") {
extraction.attributes = data[attributesKey];
}
extractions.push(extraction);
}
}
}
return extractions;
}
alignSingleExtraction(extraction, sourceTokens, sourceText, tokenOffset, charOffset, enableFuzzyAlignment, fuzzyAlignmentThreshold) {
const extractionTokens = (0, tokenizer_1.tokenize)(extraction.extractionText).tokens;
// Try exact match first
const exactMatch = this.findExactMatch(extractionTokens, sourceTokens, tokenOffset);
if (exactMatch) {
return {
...extraction,
charInterval: this.tokenToCharInterval(exactMatch.start, exactMatch.end, sourceText, charOffset),
alignmentStatus: types_1.AlignmentStatus.MATCH_EXACT,
};
}
// Try fuzzy alignment if enabled
if (enableFuzzyAlignment) {
const fuzzyMatch = this.findFuzzyMatch(extractionTokens, sourceTokens, tokenOffset, fuzzyAlignmentThreshold);
if (fuzzyMatch) {
return {
...extraction,
charInterval: this.tokenToCharInterval(fuzzyMatch.start, fuzzyMatch.end, sourceText, charOffset),
alignmentStatus: types_1.AlignmentStatus.MATCH_FUZZY,
};
}
}
return null;
}
findExactMatch(extractionTokens, sourceTokens, tokenOffset) {
const normalizedExtraction = extractionTokens.map(tokenizer_1.normalizeToken);
for (let i = tokenOffset; i <= sourceTokens.length - normalizedExtraction.length; i++) {
let match = true;
for (let j = 0; j < normalizedExtraction.length; j++) {
if ((0, tokenizer_1.normalizeToken)(sourceTokens[i + j]) !== normalizedExtraction[j]) {
match = false;
break;
}
}
if (match) {
return { start: i, end: i + normalizedExtraction.length };
}
}
return null;
}
findFuzzyMatch(extractionTokens, sourceTokens, tokenOffset, threshold) {
// Simplified fuzzy matching - in a real implementation, you'd use more sophisticated algorithms
const normalizedExtraction = extractionTokens.map(tokenizer_1.normalizeToken);
for (let i = tokenOffset; i <= sourceTokens.length - normalizedExtraction.length; i++) {
let matches = 0;
for (let j = 0; j < normalizedExtraction.length; j++) {
if ((0, tokenizer_1.normalizeToken)(sourceTokens[i + j]) === normalizedExtraction[j]) {
matches++;
}
}
const similarity = matches / normalizedExtraction.length;
if (similarity >= threshold) {
return { start: i, end: i + normalizedExtraction.length };
}
}
return null;
}
tokenToCharInterval(startToken, endToken, sourceText, charOffset) {
const tokens = (0, tokenizer_1.tokenize)(sourceText);
let startPos = charOffset;
let endPos = charOffset;
if (startToken < tokens.charIntervals.length) {
startPos += tokens.charIntervals[startToken].startPos ?? 0;
}
if (endToken <= tokens.charIntervals.length) {
endPos += tokens.charIntervals[endToken - 1].endPos ?? sourceText.length;
}
else {
endPos += sourceText.length;
}
return { startPos, endPos };
}
}
exports.Resolver = Resolver;
//# sourceMappingURL=resolver.js.map