langextract
Version:
A TypeScript library for extracting structured and grounded information from text using LLMs
69 lines • 2.17 kB
JavaScript
;
/**
* Copyright 2025 kmbro.
*
* This is a TypeScript translation of the original Python LangExtract library
* by Google LLC (https://github.com/google/langextract).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenize = tokenize;
exports.normalizeToken = normalizeToken;
exports.tokenizeWithLowercase = tokenizeWithLowercase;
/**
* Tokenizes text into words and tracks character positions.
* This is a simplified tokenizer that splits on whitespace and punctuation.
*/
function tokenize(text) {
const tokens = [];
const tokenIntervals = [];
const charIntervals = [];
// Simple tokenization: split on whitespace and punctuation
const tokenRegex = /\b\w+\b|[^\w\s]/g;
let match;
let tokenIndex = 0;
while ((match = tokenRegex.exec(text)) !== null) {
const token = match[0];
const startPos = match.index;
const endPos = startPos + token.length;
tokens.push(token);
tokenIntervals.push({
startToken: tokenIndex,
endToken: tokenIndex + 1,
});
charIntervals.push({
startPos,
endPos,
});
tokenIndex++;
}
return {
tokens,
tokenIntervals,
charIntervals,
};
}
/**
* Normalizes a token for comparison (lowercase, trim).
*/
function normalizeToken(token) {
return token.toLowerCase().trim();
}
/**
* Tokenizes text with lowercase normalization.
*/
function tokenizeWithLowercase(text) {
return tokenize(text).tokens.map(normalizeToken);
}
//# sourceMappingURL=tokenizer.js.map