@flavoai/fastfold
Version:
Flavo frontend package
667 lines • 25.2 kB
JavaScript
// ============================================================================
// FASTFOLD IN-APP ANALYTICS - Buffered Aggregation
// ============================================================================
//
// Memory-efficient analytics for resource-constrained containers (500MB RAM).
// Events are buffered in memory and flushed every 30 seconds or when full.
//
// ============================================================================
import { createHash } from 'crypto';
import { eq, sql } from 'drizzle-orm';
// ============================================================================
// REFERRER PARSING
// ============================================================================
const REFERRER_PATTERNS = [
[/facebook\.com|fb\.com|fbcdn/, 'facebook'],
[/twitter\.com|t\.co|x\.com/, 'twitter'],
[/instagram\.com/, 'instagram'],
[/google\.|googleapis/, 'google'],
[/bing\.com/, 'bing'],
[/linkedin\.com/, 'linkedin'],
[/youtube\.com/, 'youtube'],
[/reddit\.com/, 'reddit'],
[/tiktok\.com/, 'tiktok'],
[/pinterest\.com/, 'pinterest'],
];
function parseReferrer(referrer) {
if (!referrer)
return 'direct';
for (const [pattern, source] of REFERRER_PATTERNS) {
if (pattern.test(referrer))
return source;
}
return 'other';
}
// ============================================================================
// ERROR FINGERPRINTING
// ============================================================================
function createErrorFingerprint(event) {
const parts = [
event.error?.message || '',
event.error?.component || '',
event.error?.endpoint || '',
event.event_type,
];
return createHash('sha256')
.update(parts.join('|'))
.digest('hex')
.slice(0, 32);
}
// ============================================================================
// ANALYTICS BUFFER MANAGER
// ============================================================================
export class AnalyticsBufferManager {
eventBuffer = [];
sessionMap = new Map();
flushTimer = null;
sessionCleanupTimer = null;
isShuttingDown = false;
isFlushing = false; // Prevent concurrent flushes
shutdownHandlerRegistered = false;
db = null;
schema = null;
// Rate limiting for incoming events
eventCountLastMinute = 0;
lastMinuteReset = Date.now();
static MAX_EVENTS_PER_MINUTE = 1000;
// Session cleanup - remove sessions older than 1 hour
static SESSION_TIMEOUT_MS = 60 * 60 * 1000;
// Max stack trace length to store
static MAX_STACK_LENGTH = 2000;
config = {
flushIntervalMs: 30000,
maxBufferSize: 50,
enabled: true,
};
constructor(config) {
if (config) {
this.config = { ...this.config, ...config };
}
}
/**
* Initialize with Drizzle database and schema
*/
initialize(db, schema) {
this.db = db;
this.schema = schema;
if (this.config.enabled) {
this.startFlushTimer();
this.startSessionCleanupTimer();
this.setupShutdownHandler();
console.log(`[analytics] Buffer initialized (flush every ${this.config.flushIntervalMs / 1000}s)`);
}
}
/**
* Start timer to cleanup stale sessions (every 5 minutes)
*/
startSessionCleanupTimer() {
if (this.sessionCleanupTimer) {
clearInterval(this.sessionCleanupTimer);
}
this.sessionCleanupTimer = setInterval(() => {
this.cleanupStaleSessions();
}, 5 * 60 * 1000); // Every 5 minutes
}
/**
* Remove sessions older than SESSION_TIMEOUT_MS
*/
cleanupStaleSessions() {
const now = Date.now();
let cleaned = 0;
for (const [sessionId, data] of this.sessionMap) {
if (now - data.createdAt > AnalyticsBufferManager.SESSION_TIMEOUT_MS) {
this.sessionMap.delete(sessionId);
cleaned++;
}
}
if (cleaned > 0) {
console.log(`[analytics] Cleaned up ${cleaned} stale sessions`);
}
}
/**
* Add event to buffer with rate limiting and validation
*/
addEvent(event) {
if (!this.config.enabled)
return;
// Rate limiting - reset counter every minute
const now = Date.now();
if (now - this.lastMinuteReset > 60000) {
this.eventCountLastMinute = 0;
this.lastMinuteReset = now;
}
// Check rate limit
if (this.eventCountLastMinute >= AnalyticsBufferManager.MAX_EVENTS_PER_MINUTE) {
return; // Drop event silently
}
// Validate required fields
if (!this.validateEvent(event)) {
return;
}
// Truncate stack traces to prevent huge JSON
if (event.error?.stack) {
event.error.stack = event.error.stack.slice(0, AnalyticsBufferManager.MAX_STACK_LENGTH);
}
this.eventBuffer.push(event);
this.eventCountLastMinute++;
// Flush if buffer is full
if (this.eventBuffer.length >= this.config.maxBufferSize) {
this.flush().catch(console.error);
}
}
/**
* Validate event has required fields
*/
validateEvent(event) {
if (!event.visitor_id || !event.session_id) {
return false;
}
if (!event.event_type) {
return false;
}
if (!event.timestamp) {
// Auto-fill timestamp if missing
event.timestamp = new Date().toISOString();
}
return true;
}
/**
* Add multiple events to buffer
*/
addEvents(events) {
for (const event of events) {
this.addEvent(event);
}
}
/**
* Start the periodic flush timer
*/
startFlushTimer() {
if (this.flushTimer) {
clearInterval(this.flushTimer);
}
this.flushTimer = setInterval(() => {
this.flush().catch(console.error);
}, this.config.flushIntervalMs);
}
/**
* Setup graceful shutdown handler (only once)
*/
setupShutdownHandler() {
if (this.shutdownHandlerRegistered)
return;
this.shutdownHandlerRegistered = true;
const shutdown = async () => {
if (this.isShuttingDown)
return;
this.isShuttingDown = true;
console.log('[analytics] Graceful shutdown - flushing buffer...');
await this.flush();
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = null;
}
if (this.sessionCleanupTimer) {
clearInterval(this.sessionCleanupTimer);
this.sessionCleanupTimer = null;
}
};
process.on('SIGTERM', shutdown);
process.on('SIGINT', shutdown);
}
/**
* Flush buffered events to database (with concurrency protection)
*/
async flush() {
// Prevent concurrent flushes
if (this.isFlushing) {
return { analytics: 0, errors: 0 };
}
if (this.eventBuffer.length === 0) {
return { analytics: 0, errors: 0 };
}
if (!this.db || !this.schema) {
console.warn('[analytics] Database not initialized, skipping flush');
return { analytics: 0, errors: 0 };
}
this.isFlushing = true;
// Take events from buffer atomically
const events = [...this.eventBuffer];
this.eventBuffer = [];
console.log(`[analytics] Flushing ${events.length} events`);
try {
const result = await this.processEvents(events);
return result;
}
catch (error) {
console.error('[analytics] Flush error:', error);
// Put events back in buffer for retry (but limit to avoid memory issues)
if (this.eventBuffer.length < this.config.maxBufferSize * 2) {
this.eventBuffer = [...events, ...this.eventBuffer];
}
return { analytics: 0, errors: 0 };
}
finally {
this.isFlushing = false;
}
}
/**
* Process events and aggregate into database
*/
async processEvents(events) {
// Group by date
const analyticsMap = new Map();
const errorMap = new Map();
for (const event of events) {
const date = event.timestamp
? event.timestamp.split('T')[0]
: new Date().toISOString().split('T')[0];
// Get or create analytics stats for this date
let stats = analyticsMap.get(date);
if (!stats) {
stats = {
date,
visitorIds: {},
sessionIds: {},
userIds: {},
pages: {},
geoBreakdown: {},
referrers: {},
customEvents: {},
pageviews: 0,
bounces: 0,
totalDuration: 0,
};
analyticsMap.set(date, stats);
}
// Process based on event type
switch (event.event_type) {
case 'pageview': {
stats.visitorIds[event.visitor_id] = 1;
stats.sessionIds[event.session_id] = 1;
stats.pageviews++;
// Track page
const pagePath = event.page_path || this.extractPath(event.url) || '/';
stats.pages[pagePath] = (stats.pages[pagePath] || 0) + 1;
// Track geo
if (event.geo?.country) {
const country = event.geo.country;
if (!stats.geoBreakdown[country]) {
stats.geoBreakdown[country] = {
count: 0,
lat: event.geo.lat,
lng: event.geo.lng,
};
}
stats.geoBreakdown[country].count++;
}
// Track logged-in user
if (event.user_id) {
if (!stats.userIds[event.user_id]) {
stats.userIds[event.user_id] = {
visits: 0,
lastSeen: event.timestamp,
email: event.user_email,
};
}
stats.userIds[event.user_id].visits++;
stats.userIds[event.user_id].lastSeen = event.timestamp;
}
break;
}
case 'session_start': {
stats.visitorIds[event.visitor_id] = 1;
stats.sessionIds[event.session_id] = 1;
// Track session for duration/bounce calculation (with createdAt for cleanup)
this.sessionMap.set(event.session_id, {
pageviews: 0,
startTime: new Date(event.timestamp).getTime(),
createdAt: Date.now(),
});
// Track referrer
const source = parseReferrer(event.referrer);
stats.referrers[source] = (stats.referrers[source] || 0) + 1;
break;
}
case 'session_end': {
const session = this.sessionMap.get(event.session_id);
if (session) {
session.endTime = new Date(event.timestamp).getTime();
session.pageviewCount = event.pageview_count || 1;
// Calculate bounce (single pageview session)
if ((event.pageview_count || 1) === 1) {
stats.bounces++;
}
// Calculate duration
const duration = event.properties?.duration_seconds ||
Math.round((session.endTime - session.startTime) / 1000);
if (duration > 0) {
stats.totalDuration += duration;
}
// Cleanup session map
this.sessionMap.delete(event.session_id);
}
break;
}
case 'error_frontend':
case 'error_backend':
case 'error_api': {
const fingerprint = createErrorFingerprint(event);
const existing = errorMap.get(fingerprint);
if (!existing) {
errorMap.set(fingerprint, {
fingerprint,
errorType: event.event_type === 'error_backend'
? 'backend'
: event.event_type === 'error_api'
? 'api'
: 'frontend',
severity: event.error?.severity || 'error',
message: event.error?.message || 'Unknown error',
stack: event.error?.stack,
component: event.error?.component,
endpoint: event.error?.endpoint,
statusCode: event.error?.status_code,
count: 1,
});
}
else {
existing.count++;
}
break;
}
case 'custom': {
if (event.event_name) {
stats.customEvents[event.event_name] =
(stats.customEvents[event.event_name] || 0) + 1;
}
break;
}
}
}
// Write to database
let analyticsCount = 0;
let errorsCount = 0;
// Upsert analytics daily
for (const [date, stats] of analyticsMap) {
await this.upsertAnalyticsDaily(date, stats);
analyticsCount++;
}
// Upsert errors
for (const [, errorData] of errorMap) {
await this.upsertError(errorData);
errorsCount++;
}
return { analytics: analyticsCount, errors: errorsCount };
}
/**
* Extract path from URL
*/
extractPath(url) {
if (!url)
return null;
try {
return new URL(url).pathname;
}
catch {
return null;
}
}
/**
* Upsert analytics daily record (with table existence check)
*/
async upsertAnalyticsDaily(date, stats) {
const { analyticsDaily } = this.schema;
if (!analyticsDaily) {
// Table not in schema - likely old app without migration
// Skip silently to avoid spamming logs
return;
}
try {
// Check if record exists
const existing = await this.db
.select()
.from(analyticsDaily)
.where(eq(analyticsDaily.date, date))
.limit(1);
// Limit JSON field sizes to prevent unbounded growth
const visitorIds = this.limitJsonKeys(stats.visitorIds, 10000);
const sessionIds = this.limitJsonKeys(stats.sessionIds, 10000);
const userIds = this.limitJsonKeys(stats.userIds, 1000);
const pages = this.limitJsonKeys(stats.pages, 500);
if (existing.length > 0) {
// Merge with existing
const current = existing[0];
const merged = this.mergeAnalytics(current, {
...stats,
visitorIds,
sessionIds,
userIds,
pages,
});
await this.db
.update(analyticsDaily)
.set({
visitorsCount: merged.visitorsCount,
pageviewsCount: merged.pageviewsCount,
sessionsCount: merged.sessionsCount,
bouncesCount: merged.bouncesCount,
totalDurationSeconds: merged.totalDurationSeconds,
visitorIds: merged.visitorIds,
sessionIds: merged.sessionIds,
pages: merged.pages,
referrers: merged.referrers,
geoBreakdown: merged.geoBreakdown,
customEvents: merged.customEvents,
userIds: merged.userIds,
})
.where(eq(analyticsDaily.date, date));
}
else {
// Insert new record
await this.db.insert(analyticsDaily).values({
date,
visitorsCount: Object.keys(visitorIds).length,
pageviewsCount: stats.pageviews,
sessionsCount: Object.keys(sessionIds).length,
bouncesCount: stats.bounces,
totalDurationSeconds: stats.totalDuration,
visitorIds: JSON.stringify(visitorIds),
sessionIds: JSON.stringify(sessionIds),
pages: JSON.stringify(pages),
referrers: JSON.stringify(stats.referrers),
geoBreakdown: JSON.stringify(stats.geoBreakdown),
customEvents: JSON.stringify(stats.customEvents),
userIds: JSON.stringify(userIds),
});
}
}
catch (error) {
// Check if it's a "table doesn't exist" error (SQLite)
if (error?.message?.includes('no such table')) {
// Table doesn't exist yet - skip silently
return;
}
console.error('[analytics] Failed to upsert analytics daily:', error);
}
}
/**
* Limit number of keys in a JSON object to prevent unbounded growth
*/
limitJsonKeys(obj, maxKeys) {
const keys = Object.keys(obj);
if (keys.length <= maxKeys) {
return obj;
}
// Keep the most recent keys (assumes keys are added in order)
const result = {};
const keysToKeep = keys.slice(-maxKeys);
for (const key of keysToKeep) {
result[key] = obj[key];
}
return result;
}
/**
* Merge existing analytics with new stats
*/
mergeAnalytics(existing, stats) {
const parseJSON = (val) => {
if (!val)
return {};
if (typeof val === 'object')
return val;
try {
return JSON.parse(val);
}
catch {
return {};
}
};
const mergeObjects = (a, b) => {
const result = { ...a };
for (const [key, value] of Object.entries(b)) {
if (typeof value === 'number') {
result[key] = (result[key] || 0) + value;
}
else if (typeof value === 'object' && value !== null) {
result[key] = { ...result[key], ...value };
}
else {
result[key] = value;
}
}
return result;
};
const existingVisitorIds = parseJSON(existing.visitorIds);
const existingSessionIds = parseJSON(existing.sessionIds);
const mergedVisitorIds = { ...existingVisitorIds, ...stats.visitorIds };
const mergedSessionIds = { ...existingSessionIds, ...stats.sessionIds };
return {
visitorsCount: Object.keys(mergedVisitorIds).length,
pageviewsCount: (existing.pageviewsCount || 0) + stats.pageviews,
sessionsCount: Object.keys(mergedSessionIds).length,
bouncesCount: (existing.bouncesCount || 0) + stats.bounces,
totalDurationSeconds: (existing.totalDurationSeconds || 0) + stats.totalDuration,
visitorIds: JSON.stringify(mergedVisitorIds),
sessionIds: JSON.stringify(mergedSessionIds),
pages: JSON.stringify(mergeObjects(parseJSON(existing.pages), stats.pages)),
referrers: JSON.stringify(mergeObjects(parseJSON(existing.referrers), stats.referrers)),
geoBreakdown: JSON.stringify(mergeObjects(parseJSON(existing.geoBreakdown), stats.geoBreakdown)),
customEvents: JSON.stringify(mergeObjects(parseJSON(existing.customEvents), stats.customEvents)),
userIds: JSON.stringify(mergeObjects(parseJSON(existing.userIds), stats.userIds)),
};
}
/**
* Upsert error record (with table existence check)
*/
async upsertError(errorData) {
const { analyticsErrors } = this.schema;
if (!analyticsErrors) {
// Table not in schema - skip silently
return;
}
try {
// Truncate stack trace if too long
const truncatedStack = errorData.stack
? errorData.stack.slice(0, AnalyticsBufferManager.MAX_STACK_LENGTH)
: undefined;
// Truncate message if too long
const truncatedMessage = errorData.message.slice(0, 500);
// Check if error exists by fingerprint
const existing = await this.db
.select()
.from(analyticsErrors)
.where(eq(analyticsErrors.fingerprint, errorData.fingerprint))
.limit(1);
if (existing.length > 0) {
// Update occurrence count and last seen
await this.db
.update(analyticsErrors)
.set({
occurrenceCount: sql `${analyticsErrors.occurrenceCount} + ${errorData.count}`,
lastSeen: new Date(),
})
.where(eq(analyticsErrors.fingerprint, errorData.fingerprint));
}
else {
// Insert new error
await this.db.insert(analyticsErrors).values({
fingerprint: errorData.fingerprint,
errorType: errorData.errorType,
severity: errorData.severity,
message: truncatedMessage,
stack: truncatedStack,
component: errorData.component?.slice(0, 200),
endpoint: errorData.endpoint?.slice(0, 200),
statusCode: errorData.statusCode,
occurrenceCount: errorData.count,
firstSeen: new Date(),
lastSeen: new Date(),
resolved: false,
});
}
}
catch (error) {
// Check if it's a "table doesn't exist" error
if (error?.message?.includes('no such table')) {
return;
}
console.error('[analytics] Failed to upsert error:', error);
}
}
/**
* Get current buffer size (for monitoring)
*/
getBufferSize() {
return this.eventBuffer.length;
}
/**
* Destroy the manager
*/
async destroy() {
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = null;
}
if (this.sessionCleanupTimer) {
clearInterval(this.sessionCleanupTimer);
this.sessionCleanupTimer = null;
}
await this.flush();
this.sessionMap.clear();
}
}
// ============================================================================
// SINGLETON INSTANCE
// ============================================================================
let analyticsInstance = null;
/**
* Initialize analytics buffer manager
*/
export function initializeAnalytics(db, schema, config) {
if (analyticsInstance) {
analyticsInstance.destroy().catch(console.error);
}
analyticsInstance = new AnalyticsBufferManager(config);
analyticsInstance.initialize(db, schema);
return analyticsInstance;
}
/**
* Get the analytics buffer manager instance
*/
export function getAnalyticsManager() {
return analyticsInstance;
}
/**
* Add events to the analytics buffer
*/
export function trackEvents(events) {
analyticsInstance?.addEvents(events);
}
/**
* Force flush analytics buffer
*/
export async function flushAnalytics() {
return analyticsInstance?.flush() ?? { analytics: 0, errors: 0 };
}
//# sourceMappingURL=analytics.js.map