@convo-lang/convo-lang
Version:
The language of AI
527 lines (521 loc) • 20.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConvoDocQueryRunner = exports.enableConvoDocRunnerLogging = void 0;
const convo_lang_1 = require("@convo-lang/convo-lang");
const common_1 = require("@iyio/common");
const json5_1 = require("@iyio/json5");
const vfs_1 = require("@iyio/vfs");
const rxjs_1 = require("rxjs");
const lsKey = 'enableConvoDocRunnerLogging';
let localStorageCheckedForLogging = false;
const checkLsForLogging = () => {
localStorageCheckedForLogging = true;
if (globalThis.localStorage) {
if (globalThis.localStorage.getItem(lsKey) === 'true') {
enableLogging = true;
}
}
};
let enableLogging = false;
const enableConvoDocRunnerLogging = (enable) => {
enableLogging = enable;
};
exports.enableConvoDocRunnerLogging = enableConvoDocRunnerLogging;
if (globalThis.window) {
try {
globalThis.window.__enableConvoDocRunnerLogging = exports.enableConvoDocRunnerLogging;
}
catch { }
}
const removeFromCacheAfter = (key, ttl, log = enableLogging) => {
if (ttl <= 0) {
return;
}
setTimeout(() => {
const cached = memoryCache[key];
if (cached && cached.ttl < Date.now()) {
delete memoryCache[key];
if (log) {
console.log('Remove item from ConvoDocQueryRunner mem cache ', key);
}
}
}, ttl + 1000);
};
const memoryCache = {};
const cacheInMem = (ttl, key, result) => {
memoryCache[key] = {
r: (0, common_1.deepClone)(result),
ttl: Date.now() + ttl
};
removeFromCacheAfter(key, ttl);
};
const getFromMemCache = (key, ttl, log = enableLogging) => {
const mem = memoryCache[key];
if (mem && mem.ttl > Date.now()) {
removeFromCacheAfter(key, ttl);
if (log) {
console.log(`doc query loaded from memory - ${key}`);
}
return (0, common_1.deepClone)(mem.r);
}
else {
return undefined;
}
};
const runLock = new common_1.Lock(1);
class ConvoDocQueryRunner {
options;
_result = new rxjs_1.BehaviorSubject(null);
get resultSubject() { return this._result; }
get result() { return this._result.value; }
outputs = [];
nextOutputId = 1;
llmLock;
disposeToken = new common_1.CancelToken();
progress;
constructor({ query, llmLock = 5, createConversation, cacheQueryResults = false, cacheVisionPass = cacheQueryResults, cacheTextPass = cacheQueryResults, cacheConversations = false, conversationCache, cacheDir = '/cache/document-queries', outDir, readerFactory, memoryCacheTtlMs = cacheQueryResults ? common_1.minuteMs * 2 : 0, log = false, conversationOptions, useRunLock = false, }) {
if (!localStorageCheckedForLogging && globalThis.localStorage) {
checkLsForLogging();
}
this.options = {
query,
llmLock,
createConversation,
cacheQueryResults,
cacheConversations,
cacheTextPass,
conversationCache,
cacheVisionPass,
cacheDir,
outDir,
readerFactory,
memoryCacheTtlMs,
log,
conversationOptions,
useRunLock
};
const url = (0, vfs_1.getVfsItemUrl)(query.src);
this.progress = new common_1.Progress(`Document Query - ${(0, common_1.getFileName)(url)}`, 'Starting');
this.llmLock = new common_1.Lock(llmLock);
}
_isDisposed = false;
get isDisposed() { return this._isDisposed; }
dispose() {
if (this._isDisposed) {
return;
}
this._isDisposed = true;
this.disposeToken.cancelNow();
}
createConversation() {
return this.options.createConversation?.() ?? new convo_lang_1.Conversation({
cache: (this.options.conversationCache ??
(this.options.cacheConversations ? new convo_lang_1.ConvoLocalStorageCache() : undefined)),
...(0, common_1.dupDeleteUndefined)(this.options.conversationOptions)
});
}
pageLock = new common_1.Lock(1);
reader;
readOpenCount = 0;
async getPageImageAsync(index) {
if (!this.reader?.pageToImageAsync) {
return undefined;
}
const release = await this.pageLock.waitOrCancelAsync(this.disposeToken);
if (!release) {
return undefined;
}
this.readOpenCount++;
try {
if (!this.reader) {
return undefined;
}
return await this.reader.pageToImageAsync?.(index);
}
finally {
this.readOpenCount--;
release();
}
}
async getPageTextAsync(index) {
if (!this.reader?.pageToTextAsync) {
return undefined;
}
const release = await this.pageLock.waitOrCancelAsync(this.disposeToken);
if (!release) {
return undefined;
}
this.readOpenCount++;
try {
if (!this.reader) {
return undefined;
}
return await this.reader.pageToTextAsync?.(index);
}
finally {
this.readOpenCount--;
release();
}
}
async getCachedPassAsync(pass, query) {
const queryPass = { ...query };
queryPass.select = query.select?.filter(s => (s.pass ?? 0) <= pass);
const hashKey = (0, common_1.getSortedObjectHash)({ _: convo_lang_1.convoDocResultFormatVersion, q: queryPass });
const cachePath = (0, common_1.joinPaths)(this.options.cacheDir, hashKey + `-pass-${pass}.json`);
let outputs;
try {
outputs = await (0, vfs_1.vfs)().readObjectAsync(cachePath) ?? undefined;
if (outputs && enableLogging) {
console.log(`doc query pass ${pass} loaded from cache - ${cachePath}`);
}
}
catch (ex) {
console.error(`Failed to load doc query pass ${pass} from cached`, ex);
return {
outputs: undefined,
key: hashKey,
};
}
return {
outputs,
key: hashKey,
};
}
async writeCachedPassAsync(key, pass) {
const outputs = this.outputs.filter(o => o.pass === pass);
const cachePath = (0, common_1.joinPaths)(this.options.cacheDir, key + `-pass-${pass}.json`);
try {
await (0, vfs_1.vfs)().writeObjectAsync(cachePath, outputs);
if (enableLogging || this.options.log) {
console.info(`doc query pass ${pass} written to cache - ${cachePath}`);
}
}
catch (ex) {
console.error(`Failed to write doc query pass to cache - ${pass}`, ex);
}
}
loadCached(outputs) {
for (const o of outputs) {
if (o.id >= this.nextOutputId) {
this.nextOutputId = o.id + 1;
}
this.outputs.push(o);
}
}
progressTotal = 0;
progressStep = 0;
updateProgress(steps, status) {
this.progressStep += steps;
this.progress.set(this.progressStep / (this.progressTotal || 1), status);
}
runPromise;
async runQueryAsync() {
if (this.runPromise) {
return await this.runPromise;
}
if (this.options.useRunLock) {
const release = await runLock.waitOrCancelAsync(this.disposeToken);
if (!release) {
return { outputs: [], pages: [] };
}
try {
this.runPromise = this._runQueryAsync();
}
finally {
release();
}
}
else {
this.runPromise = this._runQueryAsync();
}
return await this.runPromise;
}
async _runQueryAsync() {
// for now assume the document is a pdf
const query = this.options.query;
const url = (0, vfs_1.getVfsItemUrl)(query.src);
if (!url) {
throw new Error("Unable to get url for src");
}
const hashKey = this.options.cacheQueryResults ? (0, common_1.getSortedObjectHash)({ _: convo_lang_1.convoDocResultFormatVersion, url, query }) : '';
const cachePathBase = (0, common_1.joinPaths)(this.options.cacheDir, hashKey);
const cachePath = cachePathBase + '.json';
if (this.options.cacheQueryResults) {
const mem = getFromMemCache(hashKey, this.options.memoryCacheTtlMs);
if (mem) {
if (enableLogging || this.options.log) {
console.log(`doc query loaded from memory - ${hashKey}`);
}
return mem;
}
const cached = memoryCache[hashKey]?.r ?? await (0, vfs_1.vfs)().readObjectAsync(cachePath);
if (cached) {
if (this.options.memoryCacheTtlMs > 0) {
cacheInMem(this.options.memoryCacheTtlMs, hashKey, cached);
}
if (enableLogging || this.options.log) {
console.log(`doc query loaded from cache - ${cachePath}`);
}
return cached;
}
}
const reader = await (0, convo_lang_1.getConvoDocReaderAsync)(query.src, this.options.readerFactory);
if (!reader) {
throw new Error(`Unable to get doc reader for query source. url - ${url}`);
}
this.reader = reader;
try {
this.updateProgress(0, 'Loading document');
const pageCount = await reader.getPageCountAsync();
const pages = [];
for (let i = 0; i < pageCount; i++) {
const page = {
index: i,
};
pages.push(page);
}
const useText = (query.textPass && reader.pageToTextAsync) ? true : false;
const useVision = (query.visionPass && reader.pageToImageAsync) ? true : false;
this.progressTotal = (useText ? pageCount : 0) + (useVision ? pageCount : 0) + (query.select?.length ?? 0);
this.progressStep = 0;
if (useText) {
this.updateProgress(0, 'Reading text');
let readCount = 0;
await this.runPassAsync(query, -2, pages, this.options.cacheTextPass, async (page) => {
const text = await this.getPageTextAsync(page.index);
readCount++;
this.updateProgress(1, `Page ${readCount} of ${pageCount} read`);
if (!text) {
return;
}
this.outputs.push({
id: this.nextOutputId++,
output: text,
contentType: 'text/plain',
type: 'content',
pass: -2,
pageIndexes: [page.index]
});
});
}
if (useVision) {
this.updateProgress(0, 'Scanning with vision');
let readCount = 0;
await this.runPassAsync(query, -1, pages, this.options.cacheVisionPass, async (page) => {
const img = await this.getPageImageAsync(page.index);
if (!img) {
readCount++;
this.updateProgress(1, `Page ${readCount} of ${pageCount} skipped`);
return;
}
await this.convertPageImageAsync(page, img);
readCount++;
this.updateProgress(1, `Page ${readCount} of ${pageCount} scanned`);
});
}
if (query.select) {
let lastPass = 0;
for (const select of query.select) {
if (select.pass && select.pass > lastPass) {
lastPass = select.pass;
}
}
for (let pass = 0; pass <= lastPass; pass++) {
let passCacheKey;
if (this.options.cacheQueryResults) {
const cached = await this.getCachedPassAsync(pass, query);
passCacheKey = cached.key;
if (cached.outputs) {
this.loadCached(cached.outputs);
continue;
}
}
const passSelects = query.select.filter(s => (s.pass ?? 0) === pass);
if (passSelects.length) {
await Promise.all(passSelects.map(s => this.selectAsync(pages.filter(p => (0, convo_lang_1.isConvoDocSelectMatch)(p.index, s)), s, pass, false)));
this.updateProgress(passSelects.length, `Pass ${pass} complete`);
}
if (passCacheKey) {
await this.writeCachedPassAsync(passCacheKey, pass);
}
}
}
const result = {
pages,
outputs: this.outputs
};
if (this.options.memoryCacheTtlMs > 0) {
cacheInMem(this.options.memoryCacheTtlMs, hashKey, result);
}
if (this.options.cacheQueryResults) {
try {
await (0, vfs_1.vfs)().writeObjectAsync(cachePath, result);
if (enableLogging || this.options.log) {
console.log(`doc query pass written to cache - ${cachePath}`);
}
}
catch (ex) {
console.error('Failed to write document query result to cache', ex);
}
}
return result;
}
finally {
reader.dispose?.();
this.reader = undefined;
}
}
async runPassAsync(query, pass, pages, cache, pageCallback) {
let loadedFromCached = false;
let passCacheKey;
if (this.options.cacheQueryResults || cache) {
const cached = await this.getCachedPassAsync(pass, query);
passCacheKey = cached.key;
if (cached.outputs) {
this.loadCached(cached.outputs);
loadedFromCached = true;
}
}
if (!loadedFromCached) {
await Promise.all(pages.map(pageCallback));
if (passCacheKey) {
await this.writeCachedPassAsync(passCacheKey, pass);
}
}
}
async convertPageImageAsync(page, img) {
const b64 = await (0, common_1.readBlobAsDataUrlAsync)(img);
const r = await this.callAsync(/*convo*/ `
> system
You are helping a user convert pages of a document into markdown documents.
You will be given each page as an image.
Convert tables, graphs and charts into markdown tables with a detailed description.
If a graph or chart can not be converted into a markdown table convert it to [Mermaid](https://mermaid.js.org/) diagram in a code block.
Convert images into markdown images with a detailed description in the alt text area and if you don't know the full URL to the image use an empty anchor link (a single hash tag).
Ignore any navigation UI elements.
Ignore headers and footers that included information that would be repeated every page.
Ignore page numbers at the top or bottom of the page.
Ignore any ads.
If a blank image is given respond with the text "BLANK" in all caps.
Do not enclose your responses in a markdown code block.
Respond with your conversation of the page verbatim. Do not give an explanation of how you converted the document or tell of any issues with the document.
> user
Convert the following page
`);
if (!r?.content || (r.content.length <= 10 && r.content.includes('BLANK'))) {
return;
}
this.outputs.push({
id: this.nextOutputId++,
output: r?.content ?? '',
contentType: 'text/markdown',
type: 'content',
pass: -1,
pageIndexes: [page.index]
});
}
async selectAsync(pages, select, pass, perPage) {
if (!pages.length) {
return;
}
if (!perPage && (0, convo_lang_1.isConvoDocSelectPerPage)(select)) {
await Promise.all(pages.map(p => this.selectAsync([p], select, pass, true)));
return;
}
const generatePrompt = select.generateConvo ?? (select.generate ? `> user\n${(0, convo_lang_1.escapeConvoMessageContent)(select.generate)}` : null);
if (!generatePrompt) {
return;
}
const info = [];
for (const page of pages) {
const outputs = this.outputs.filter(o => o.pass <= pass && o.pageIndexes.includes(page.index));
if (outputs.length) {
info.push(`Page ${page.index + 1}:\n`);
for (const out of outputs) {
info.push((out.prefix ?? `<${out.type}>`) + '\n');
info.push((0, convo_lang_1.convoScript) `${out.output}`);
info.push((out.suffix ?? `</${out.type}>`) + '\n');
info.push('\n');
}
}
}
const system = info.length ? `> system\nUse the following page information for additional context:\n` + info.join('\n') + '\n\n' : '';
const requirePrompt = select.requirementConvo ?? (select.requirement ? imageRequirementPrompt(select.requirement) : null);
if (requirePrompt) {
const r = await this.callAsync(system + requirePrompt);
if (!r?.returnValue?.accept) {
return;
}
}
const result = await this.callAsync(system + generatePrompt);
if (!result) {
return;
}
let output;
let contentType = 'text/plain';
let type = select.outputType ?? 'content';
if (result.result.message?.format === 'json' && result.result.message.content) {
output = (0, json5_1.parseJson5)(result.result.message.content);
contentType = (0, convo_lang_1.getConvoSelectContentType)(select, 'application/json');
}
else if (result.returnValue === undefined) {
output = result.content ?? '';
contentType = (0, convo_lang_1.getConvoSelectContentType)(select, 'text/plain');
}
else {
const isObject = result.returnValue === undefined || (typeof result.returnValue === 'object');
output = isObject ? result.returnValue : (result.returnValue?.toString() ?? '');
contentType = (0, convo_lang_1.getConvoSelectContentType)(select, isObject ? 'application/json' : 'text/plain');
}
const outputR = {
id: this.nextOutputId++,
prefix: select.outputPrefix,
suffix: select.outputSuffix,
pageIndexes: (select.outputTarget === 'firstPage' ?
[(pages[0]?.index ?? 0)]
: select.outputTarget === 'lastPage' ?
[(pages[pages.length - 1]?.index ?? 0)]
:
pages.map(p => p.index)),
output,
contentType,
pass,
type
};
this.outputs.push(outputR);
}
async callAsync(prompt) {
const release = await this.llmLock.waitOrCancelAsync(this.disposeToken);
if (!release) {
return undefined;
}
try {
const convo = this.createConversation();
const result = await convo.completeAsync({
append: prompt,
returnOnCalled: true,
});
return {
returnValue: result.returnValues?.[0],
content: result.message?.content,
result
};
}
finally {
release();
}
}
}
exports.ConvoDocQueryRunner = ConvoDocQueryRunner;
const imageRequirementPrompt = (msg) => /*convo*/ `
@disableAutoComplete
> accept(accept:boolean)
@call
> user
Based on the following requirements call the accept function with a true value if the image meets the requirement
<requirement>
${(0, convo_lang_1.escapeConvoMessageContent)(msg)}
</requirement>
`;
//# sourceMappingURL=ConvoDocQueryRunner.js.map