pulse-ai-utils
Version:
Utility functions and helpers for AI-powered applications
190 lines • 8.41 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const axios_1 = __importDefault(require("axios"));
const zod_1 = require("zod");
const openai_helper_1 = __importDefault(require("../helpers/openai-helper"));
const env_loader_1 = require("../config/env-loader");
const pulseSchemas_1 = require("../utils/pulseSchemas");
// Use getEnvVar to properly load from Secret Manager
let serpApiKey = (0, env_loader_1.getEnvVar)('SERP_API_KEY');
class SerpHandler {
constructor() {
// Use getEnvVar which includes API key sanitization
const apiKey = (0, env_loader_1.getEnvVar)('OPENAI_API_KEY');
const finalKey = apiKey || 'dummy-key-for-tests';
this.openaiHelper = new openai_helper_1.default(finalKey);
if (!serpApiKey) {
console.error('SERP_API_KEY not configured - getEnvVar returned:', {
exists: !!serpApiKey,
rawEnv: !!process.env.SERP_API_KEY,
sanitized: !!(0, env_loader_1.getEnvVar)('SERP_API_KEY')
});
}
}
async getSerp(req, res) {
let serpUrl = req.query.url;
const area = req.query.area;
if (!serpUrl) {
return res.status(400).json({ message: 'Missing q parameter' });
}
serpUrl = serpUrl.replace("#", "%23");
serpUrl = serpUrl + `&api_key=${serpApiKey}`;
try {
// Fetch HTML from the serpUrl
const htmlResponse = await axios_1.default.get(serpUrl);
const processedData = await this.processSerpResponse(htmlResponse, area);
res.json({
output: {
data: processedData
}
});
}
catch (err) {
console.error('Error processing serp request:', err);
res.status(500).json({ message: 'Failed to fetch SERP data' });
}
}
async getSerpWithoutUrl(req, res) {
const area = req.query.area;
const region = req.query.region;
if (!area) {
return res.status(400).json({ message: 'Missing area parameter' });
}
// Build the SERP URL with the default base
const baseUrl = 'https://pulse-269146618053.us-central1.run.app/serp?url=https%3A%2F%2Fserpapi.com%2Fsearch.json%3Fq%3D';
// Construct the query with hashtag and area
let query = `%23${encodeURIComponent(area)}`;
if (region) {
query += `%20${encodeURIComponent(region)}`;
}
const serpUrl = `${baseUrl}${query}%26tbm%3Dvid%26tbs%3Ddur%3As%2Cw%26num%3D12&api_key=${serpApiKey}`;
try {
// Fetch HTML from the constructed serpUrl
const htmlResponse = await axios_1.default.get(serpUrl);
const processedData = await this.processSerpResponse(htmlResponse, area);
res.json({
output: {
data: processedData
}
});
}
catch (err) {
console.error('Error processing serp request:', err);
res.status(500).json({ message: 'Failed to fetch SERP data' });
}
}
async processSerpResponse(htmlResponse, area) {
// Use dynamic schema loading for reel validation
const { zod: ReelSchema } = (0, pulseSchemas_1.getSchemaByCategory)('reels');
const ResponseSchema = zod_1.z.object({
results: zod_1.z.array(ReelSchema),
});
// Check if the response contains structured JSON data (from SERP API)
let structuredData = [];
// Handle different response formats
if (htmlResponse.data?.output?.data && Array.isArray(htmlResponse.data.output.data)) {
// Format: { output: { data: [...] } }
structuredData = htmlResponse.data.output.data;
console.log(`Found ${structuredData.length} items in output.data`);
}
else if (htmlResponse.data?.data && Array.isArray(htmlResponse.data.data)) {
// Format: { data: [...] }
structuredData = htmlResponse.data.data;
console.log(`Found ${structuredData.length} items in data`);
}
else if (Array.isArray(htmlResponse.data)) {
// Direct array response
structuredData = htmlResponse.data;
console.log(`Found ${structuredData.length} items as direct array`);
}
// If we have structured data, process it directly without OpenAI
if (structuredData.length > 0) {
console.log('Processing structured SERP data directly');
// Map the structured data to match our Reel schema
const results = structuredData.map((item) => {
// Handle different possible field names
const reel = {
id: item.id || item.video_id || null,
title: item.title || item.name || '',
thumbnail_url: item.thumbnail_url || item.thumbnail || item.image_url || null,
duration: item.duration || null,
source: item.source || 'Unknown',
source_url: item.source_url || item.url || item.link || null,
description: item.description || null,
area: area || null,
region: null,
timestamp: new Date().toISOString()
};
return reel;
}).filter((item) => item.title && item.source_url); // Filter out invalid items
// Validate and format results
const validResults = [];
for (const item of results) {
try {
const validatedItem = ReelSchema.parse(item);
validResults.push(validatedItem);
}
catch (err) {
console.warn('Validation failed for item:', item.title, err.message);
}
}
console.log(`Validated ${validResults.length} reels from structured data`);
// Add category to all items
const formatted = validResults.map(item => ({
...item,
category: 'reels'
}));
return formatted;
}
// Fallback to original OpenAI processing for HTML content
console.log('No structured data found, falling back to OpenAI processing');
let htmlString;
if (typeof htmlResponse.data === 'string') {
htmlString = htmlResponse.data;
}
else if (htmlResponse.data.html) {
htmlString = htmlResponse.data.html;
}
else {
htmlString = JSON.stringify(htmlResponse.data); // fallback
}
let prompt = `Provide JSON with requested schema from the provided input`;
// If area is provided, include it in the prompt for better context
if (area) {
prompt += `. Focus on content related to ${area} area.`;
}
const rawOpenAiResponse = await this.openaiHelper.fetchStructuredData({
prompt,
html: htmlString,
zodSchema: ResponseSchema,
responseFormatName: 'results',
});
const openAiResponse = ResponseSchema.parse(rawOpenAiResponse);
const replaced = await Promise.all(openAiResponse.results.map(async (item) => {
if (item) {
try {
console.log('Fetching thumbnail from Microlink for', item.source_url);
const resp = await axios_1.default.get('https://api.microlink.io', { params: { url: item.source_url } });
const microlinkThumb = resp.data?.data?.image?.url;
if (microlinkThumb) {
item.thumbnail_url = microlinkThumb;
}
}
catch (err) {
console.warn('Microlink fetch failed for', item.source_url, err.message);
}
}
return item;
}));
const formatted = replaced.map(item => ({
...item,
category: 'reels'
}));
return formatted;
}
}
exports.default = SerpHandler;
//# sourceMappingURL=serp.js.map