genkitx-pinecone
Version:
Genkit AI framework plugin for Pinecone vector database.
208 lines • 7.59 kB
JavaScript
;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var index_exports = {};
__export(index_exports, {
configurePineconeIndexer: () => configurePineconeIndexer,
configurePineconeRetriever: () => configurePineconeRetriever,
createPineconeIndex: () => createPineconeIndex,
default: () => index_default,
deletePineconeIndex: () => deletePineconeIndex,
describePineconeIndex: () => describePineconeIndex,
pinecone: () => pinecone,
pineconeIndexerRef: () => pineconeIndexerRef,
pineconeRetrieverRef: () => pineconeRetrieverRef
});
module.exports = __toCommonJS(index_exports);
var import_pinecone = require("@pinecone-database/pinecone");
var import_genkit = require("genkit");
var import_plugin = require("genkit/plugin");
var import_retriever = require("genkit/retriever");
var import_ts_md5 = require("ts-md5");
const SparseVectorSchema = import_genkit.z.object({
indices: import_genkit.z.number().array(),
values: import_genkit.z.number().array()
}).refine(
(input) => {
return input.indices.length === input.values.length;
},
{
message: "Indices and values must be of the same length"
}
);
const PineconeRetrieverOptionsSchema = import_retriever.CommonRetrieverOptionsSchema.extend({
k: import_genkit.z.number().max(1e3),
namespace: import_genkit.z.string().optional(),
filter: import_genkit.z.record(import_genkit.z.string(), import_genkit.z.any()).optional(),
// includeValues is always false
// includeMetadata is always true
sparseVector: SparseVectorSchema.optional()
});
const PineconeIndexerOptionsSchema = import_genkit.z.object({
namespace: import_genkit.z.string().optional()
});
const CONTENT_KEY = "_content";
const CONTENT_TYPE = "_contentType";
const pineconeRetrieverRef = (params) => {
return (0, import_retriever.retrieverRef)({
name: `pinecone/${params.indexId}`,
info: {
label: params.displayName ?? `Pinecone - ${params.indexId}`
},
configSchema: PineconeRetrieverOptionsSchema
});
};
const pineconeIndexerRef = (params) => {
return (0, import_retriever.indexerRef)({
name: `pinecone/${params.indexId}`,
info: {
label: params.displayName ?? `Pinecone - ${params.indexId}`
},
configSchema: PineconeIndexerOptionsSchema.optional()
});
};
function pinecone(params) {
return (0, import_plugin.genkitPlugin)("pinecone", async (ai) => {
params.map((i) => configurePineconeRetriever(ai, i));
params.map((i) => configurePineconeIndexer(ai, i));
});
}
var index_default = pinecone;
function configurePineconeRetriever(ai, params) {
const { indexId, embedder, embedderOptions } = {
...params
};
const pineconeConfig = params.clientParams ?? getDefaultConfig();
const contentKey = params.contentKey ?? params.textKey ?? CONTENT_KEY;
const pinecone2 = new import_pinecone.Pinecone(pineconeConfig);
const index = pinecone2.index(indexId);
return ai.defineRetriever(
{
name: `pinecone/${params.indexId}`,
configSchema: PineconeRetrieverOptionsSchema
},
async (content, options) => {
const queryEmbeddings = await ai.embed({
embedder,
content,
options: embedderOptions
});
const scopedIndex = !!options.namespace ? index.namespace(options.namespace) : index;
const response = await scopedIndex.query({
topK: options.k,
vector: queryEmbeddings[0].embedding,
includeValues: false,
includeMetadata: true
});
return {
documents: response.matches.map((m) => m.metadata).filter((m) => !!m).map((m) => {
const metadata = m;
return import_retriever.Document.fromData(
metadata[contentKey],
metadata[CONTENT_TYPE],
metadata.docMetadata ? JSON.parse(metadata.docMetadata) : void 0
);
})
};
}
);
}
function configurePineconeIndexer(ai, params) {
const { indexId, embedder, embedderOptions } = {
...params
};
const pineconeConfig = params.clientParams ?? getDefaultConfig();
const contentKey = params.contentKey ?? params.textKey ?? CONTENT_KEY;
const pinecone2 = new import_pinecone.Pinecone(pineconeConfig);
const index = pinecone2.index(indexId);
return ai.defineIndexer(
{
name: `pinecone/${params.indexId}`,
configSchema: PineconeIndexerOptionsSchema.optional()
},
async (docs, options) => {
const scopedIndex = !!options?.namespace ? index.namespace(options.namespace) : index;
const embeddings = await Promise.all(
docs.map(
(doc) => ai.embed({
embedder,
content: doc,
options: embedderOptions
})
)
);
await scopedIndex.upsert(
embeddings.map((value, i) => {
const doc = docs[i];
const docEmbeddings = value;
const embeddingDocs = doc.getEmbeddingDocuments(docEmbeddings);
return docEmbeddings.map((docEmbedding, j) => {
const metadata = {
docMetadata: JSON.stringify(embeddingDocs[j].metadata)
};
metadata[contentKey] = embeddingDocs[j].data;
metadata[CONTENT_TYPE] = embeddingDocs[j].dataType || "";
const id = import_ts_md5.Md5.hashStr(JSON.stringify(embeddingDocs[j]));
return {
id,
values: docEmbedding.embedding,
metadata
};
});
}).reduce((acc, val) => {
return acc.concat(val);
}, [])
);
}
);
}
async function createPineconeIndex(params) {
const pineconeConfig = params.clientParams ?? getDefaultConfig();
const pinecone2 = new import_pinecone.Pinecone(pineconeConfig);
return await pinecone2.createIndex(params.options);
}
async function describePineconeIndex(params) {
const pineconeConfig = params.clientParams ?? getDefaultConfig();
const pinecone2 = new import_pinecone.Pinecone(pineconeConfig);
return await pinecone2.describeIndex(params.name);
}
async function deletePineconeIndex(params) {
const pineconeConfig = params.clientParams ?? getDefaultConfig();
const pinecone2 = new import_pinecone.Pinecone(pineconeConfig);
return await pinecone2.deleteIndex(params.name);
}
function getDefaultConfig() {
const maybeApiKey = process.env.PINECONE_API_KEY;
if (!maybeApiKey)
throw new Error(
"Please pass in the API key or set PINECONE_API_KEY environment variable.\nFor more details see https://genkit.dev/docs/plugins/pinecone"
);
return { apiKey: maybeApiKey };
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
configurePineconeIndexer,
configurePineconeRetriever,
createPineconeIndex,
deletePineconeIndex,
describePineconeIndex,
pinecone,
pineconeIndexerRef,
pineconeRetrieverRef
});
//# sourceMappingURL=index.js.map