pulse-ai-utils
Version:
Utility functions and helpers for AI-powered applications
239 lines • 9.61 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.syncContentToSupabase = syncContentToSupabase;
exports.batchSyncContent = batchSyncContent;
exports.syncQueryCacheToSupabase = syncQueryCacheToSupabase;
const index_1 = require("./index");
const openai_helper_1 = __importDefault(require("../helpers/openai-helper"));
const sanitizeId_1 = require("../utils/sanitizeId");
const vector_helpers_1 = require("./vector-helpers");
const content_types_config_1 = require("../config/content-types-config");
/**
* Content type detection using dynamic configuration
*/
function detectContentType(data) {
// Use dynamic detection from config
const detectedType = (0, content_types_config_1.detectContentTypeFromData)(data);
if (detectedType) {
// Convert to supabase format
return (0, content_types_config_1.getSupabaseTypeForContentType)(detectedType);
}
// Fallback to manual detection for legacy data
// Check for explicit itemType field (used in Flyer items)
if (data.itemType) {
return data.itemType.toLowerCase().replace('_', '-');
}
// Detect based on unique fields (legacy fallback)
if (data.product !== undefined)
return 'deal';
if (data.location !== undefined && data.date !== undefined)
return 'event';
if (data.duration !== undefined && data.thumbnail_url !== undefined)
return 'reel';
if (data.items && Array.isArray(data.items))
return 'flyer';
// Default to news_article if has standard fields
if (data.title && data.description && data.source)
return 'news_article';
throw new Error('Unable to detect content type');
}
/**
* Split text into chunks following n8n pattern (4000 chars with 500 overlap)
*/
function splitTextIntoChunks(text, chunkSize = 4000, overlap = 500) {
const chunks = [];
let start = 0;
while (start < text.length) {
const end = Math.min(start + chunkSize, text.length);
chunks.push(text.slice(start, end));
start = end - overlap;
// Prevent infinite loop on small texts
if (start >= text.length - overlap)
break;
}
return chunks;
}
/**
* Generate embeddings for content chunks
*/
async function createContentChunks(contentId, searchText, metadata, openai) {
const chunks = splitTextIntoChunks(searchText);
for (let i = 0; i < chunks.length; i++) {
const chunkText = chunks[i];
const embedding = await openai.generateEmbedding(chunkText);
const { error } = await index_1.supabase.from('content_chunks').insert({
content_id: contentId,
chunk_index: i,
chunk_text: chunkText,
chunk_embedding: (0, vector_helpers_1.formatVectorForInsert)(embedding),
metadata: metadata
});
if (error) {
console.error(`Error inserting chunk ${i} for content ${contentId}:`, error);
}
}
}
/**
* Sync a single content document from Firestore to Supabase
*/
async function syncContentToSupabase(contentId, data, isDelete = false) {
try {
// Handle deletion
if (isDelete) {
const { error } = await index_1.supabase
.from('content')
.delete()
.eq('id', contentId);
if (error) {
console.error(`Error deleting content ${contentId}:`, error);
}
return;
}
// Initialize OpenAI helper for embeddings
const openai = new openai_helper_1.default();
// Determine content type
const contentType = detectContentType(data);
// Build searchable text from all relevant fields
const textParts = [
data.title,
data.description,
data.product,
data.location,
data.source,
data.discount,
data.price
].filter(Boolean);
const searchText = textParts.join(' ');
// Generate embedding for the main content
const embedding = await openai.generateEmbedding(searchText);
// Prepare data for Supabase
const supabaseData = {
id: contentId,
type: contentType,
title: data.title || '',
source: data.source || null,
source_url: data.source_url || null,
image_url: data.image_url || data.thumbnail_url || null,
data: data, // Store full data as JSONB
seen: data.metadata?.seen || data.seen || false,
rank: data.metadata?.rank || data.rank || null,
content_date: data.date || data.timestamp || null,
embedding: (0, vector_helpers_1.formatVectorForInsert)(embedding)
};
// Upsert to Supabase
const { error } = await index_1.supabase
.from('content')
.upsert(supabaseData);
if (error) {
console.error(`Error upserting content ${contentId}:`, error);
throw error;
}
// Handle content chunks if text is long (following n8n pattern)
if (searchText.length > 4000) {
// Prepare metadata matching n8n workflow pattern
const chunkMetadata = {
category: data.category,
image_url: data.image_url,
source: data.source,
source_url: data.source_url,
date: data.date,
timestamp: new Date().toISOString()
};
await createContentChunks(contentId, searchText, chunkMetadata, openai);
}
// Handle Flyer items (sync individual items)
if (contentType === 'flyer' && data.items && Array.isArray(data.items)) {
for (let i = 0; i < data.items.length; i++) {
const item = data.items[i];
// Generate smart ID for flyer item
// If item has location data, use title+location; otherwise fallback to parent-item pattern
let itemId;
if (item.title && item.location) {
itemId = (0, sanitizeId_1.generateContentId)({
title: item.title,
location: item.location,
source_url: item.source_url || data.source_url,
page: item.page || data.page,
category: item.category || data.category,
type: item.type || 'flyer_item'
});
}
else {
// Fallback to original pattern for items without location
itemId = (0, sanitizeId_1.sanitizeId)(`${contentId}-item-${i}`);
}
// Recursively sync each item
await syncContentToSupabase(itemId, item);
// Create flyer-item relationship
const { error: linkError } = await index_1.supabase
.from('flyer_items')
.upsert({
flyer_id: contentId,
item_id: itemId,
item_order: i
});
if (linkError) {
console.error(`Error linking flyer item ${itemId}:`, linkError);
}
}
}
console.log(`Successfully synced content ${contentId} of type ${contentType}`);
}
catch (error) {
console.error(`Error syncing content ${contentId}:`, error);
throw error;
}
}
/**
* Batch sync multiple documents
*/
async function batchSyncContent(documents) {
console.log(`Starting batch sync of ${documents.length} documents`);
const results = await Promise.allSettled(documents.map(doc => syncContentToSupabase(doc.id, doc.data)));
const successful = results.filter(r => r.status === 'fulfilled').length;
const failed = results.filter(r => r.status === 'rejected').length;
console.log(`Batch sync complete: ${successful} successful, ${failed} failed`);
if (failed > 0) {
const errors = results
.filter(r => r.status === 'rejected')
.map((r, i) => ({
docId: documents[i].id,
error: r.reason
}));
console.error('Failed documents:', errors);
}
}
/**
* Sync query cache entry to Supabase
*/
async function syncQueryCacheToSupabase(cacheData) {
try {
const openai = new openai_helper_1.default();
// Generate embedding for the prompt
const promptEmbedding = await openai.generateEmbedding(cacheData.prompt);
const { error } = await index_1.supabase.from('query_cache').insert({
prompt: cacheData.prompt,
area: cacheData.area,
region: cacheData.region || null,
country: cacheData.country || null,
timeline: cacheData.timeline || null,
count: cacheData.count || null,
result: cacheData.result,
timestamp: cacheData.timestamp,
expire_at: cacheData.expireAt,
prompt_embedding: (0, vector_helpers_1.formatVectorForInsert)(promptEmbedding)
});
if (error) {
console.error('Error syncing query cache to Supabase:', error);
throw error;
}
}
catch (error) {
console.error('Error in syncQueryCacheToSupabase:', error);
throw error;
}
}
//# sourceMappingURL=firestore-sync.js.map