@podx/core/src/convex/index.ts

🚀 Core utilities and shared functionality for PODx - Advanced Twitter/X scraping and crypto analysis toolkit

// Mock API for development when Convex isn't set up
const mockApi = {
  cryptoAnalysis: {
    saveAnalysis: () => Promise.resolve('mock_id'),
    getLatestAnalysis: () => Promise.resolve(null),
    getTopSignals: () => Promise.resolve([]),
    getTweets: () => Promise.resolve([]),
    getAnalysis: () => Promise.resolve([]),
    getAccounts: () => Promise.resolve([]),
    getTrendingTokens: () => Promise.resolve([])
  },
  tokens: {
    saveTokenAnalysis: () => Promise.resolve('mock_token_id')
  },
  accounts: {
    saveAccountReputation: () => Promise.resolve('mock_account_id')
  }
};

// Avoid importing Convex types or generated API at compile time
async function getConvexApi(): Promise<any> {
  try {
    // Use dynamic import with string to avoid TypeScript compile-time checks
    const mod = await import(/* webpackIgnore: true */ '../../convex/_generated/api' + '.js');
    return (mod as any).api;
  } catch (error) {
    // Generated API not available - return mock for development
    return mockApi;
  }
}
import { promises as fs } from 'fs';
import path from 'path';

// Local database path
const LOCAL_DB_PATH = path.join(process.cwd(), 'local_data');
const LOCAL_DB_FILE = path.join(LOCAL_DB_PATH, 'database.json');

// Initialize storage based on environment
const isProduction = process.env.NODE_ENV === 'production';
const hasConvexUrl = process.env.CONVEX_URL;

let convex: any | null = null;

// Initialize Convex client lazily
async function initializeConvex() {
  if (isProduction && hasConvexUrl && !convex) {
    try {
      const ConvexHttpClient = (await import('convex/browser')).ConvexHttpClient as any;
      convex = new ConvexHttpClient(process.env.CONVEX_URL!);
    } catch (error) {
      console.warn('Failed to initialize Convex client:', error);
    }
  }
}

export class ConvexStorage {
  private client: any | null;
  private localData: any = {};

  constructor() {
    this.client = convex;
    if (!isProduction || !hasConvexUrl) {
      this.initializeLocalStorage();
    } else {
      // Initialize Convex client if needed
      initializeConvex().then(() => {
        this.client = convex;
      });
    }
  }

  private async initializeLocalStorage() {
    try {
      await fs.mkdir(LOCAL_DB_PATH, { recursive: true });
      try {
        const data = await fs.readFile(LOCAL_DB_FILE, 'utf8');
        this.localData = JSON.parse(data);
      } catch (error) {
        // File doesn't exist or is corrupted, start fresh
        this.localData = {
          cryptoAnalysis: [],
          tokens: [],
          accounts: [],
          tweets: []
        };
        await this.saveLocalData();
      }
    } catch (error) {
      console.error('Failed to initialize local storage:', error);
    }
  }

  private async saveLocalData() {
    try {
      await fs.writeFile(LOCAL_DB_FILE, JSON.stringify(this.localData, null, 2));
    } catch (error) {
      console.error('Failed to save local data:', error);
    }
  }

  // Save crypto analysis to database
  async saveAnalysis(analysisData: {
    generatedAt: string;
    totalTweets: number;
    totalAccounts: number;
    summary: any;
    tokenMentions: any;
    signals: any;
    accountProfiles: any;
  }): Promise<string> {
    if (this.client) {
      // Production: Use Convex
      try {
        const api = await getConvexApi();
        const analysisId = await this.client.mutation(api.cryptoAnalysis.saveAnalysis, analysisData);
        console.log(`✅ Analysis saved to Convex with ID: ${analysisId}`);
        return analysisId;
      } catch (error) {
        console.error('❌ Failed to save analysis to Convex:', error);
        throw error;
      }
    } else {
      // Development: Use local storage
      try {
        const analysisId = `local_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
        const analysis = { id: analysisId, ...analysisData };

        if (!this.localData.cryptoAnalysis) {
          this.localData.cryptoAnalysis = [];
        }
        this.localData.cryptoAnalysis.push(analysis);
        await this.saveLocalData();

        console.log(`✅ Analysis saved to local database with ID: ${analysisId}`);
        return analysisId;
      } catch (error) {
        console.error('❌ Failed to save analysis to local database:', error);
        throw error;
      }
    }
  }

  // Save individual token analysis
  async saveTokenAnalysis(tokenData: {
    symbol: string;
    lastUpdated: string;
    totalMentions: number;
    sentimentScore: number;
    sentiment: string;
    botScore: number;
    shillProbability: number;
    avgEngagement: number;
    uniqueAccounts: number;
    recentTweets: string[];
    signals: any;
  }): Promise<string> {
    if (this.client) {
      try {
        const api = await getConvexApi();
        const tokenId = await this.client.mutation(api.tokens.saveTokenAnalysis, tokenData);
        console.log(`✅ Token ${tokenData.symbol} saved to Convex`);
        return tokenId;
      } catch (error) {
        console.error(`❌ Failed to save token ${tokenData.symbol} to Convex:`, error);
        throw error;
      }
    } else {
      // Local storage
      if (!this.localData.tokens) this.localData.tokens = [];
      this.localData.tokens.push(tokenData);
      await this.saveLocalData();
      return `local_token_${Date.now()}`;
    }
  }

  // Save account reputation
  async saveAccountReputation(accountData: {
    username: string;
    lastUpdated: string;
    tweetCount: number;
    tokensPosted: string[];
    avgEngagement: number;
    botScore: number;
    shillScore: number;
    reputation: 'trusted' | 'suspicious' | 'bot';
    totalMentions: number;
    followerCount?: number;
    accountAge?: number;
  }): Promise<string> {
    if (this.client) {
      try {
        const api = await getConvexApi();
        const accountId = await this.client.mutation(api.accounts.saveAccountReputation, accountData);
        console.log(`✅ Account ${accountData.username} saved to Convex`);
        return accountId;
      } catch (error) {
        console.error(`❌ Failed to save account ${accountData.username} to Convex:`, error);
        throw error;
      }
    } else {
      if (!this.localData.accounts) this.localData.accounts = [];
      this.localData.accounts.push(accountData);
      await this.saveLocalData();
      return `local_account_${Date.now()}`;
    }
  }

  // Get latest analysis from database
  async getLatestAnalysis(): Promise<any> {
    if (this.client) {
      // Production: Use Convex
      try {
        const api = await getConvexApi();
        return await this.client!.query(api.cryptoAnalysis.getLatestAnalysis);
      } catch (error) {
        console.error('❌ Failed to fetch latest analysis:', error);
        throw error;
      }
    } else {
      // Development: Use local storage
      try {
        if (!this.localData.cryptoAnalysis || this.localData.cryptoAnalysis.length === 0) {
          return null;
        }

        // Get the most recent analysis
        const sorted = this.localData.cryptoAnalysis.sort((a: any, b: any) =>
          new Date(b.generatedAt).getTime() - new Date(a.generatedAt).getTime()
        );

        return sorted[0];
      } catch (error) {
        console.error('❌ Failed to fetch latest analysis from local database:', error);
        throw error;
      }
    }
  }

  // Get top signals
  async getTopSignals(limit = 5): Promise<any> {
    if (this.client) {
      // Production: Use Convex
      try {
        const api = await getConvexApi();
        return await this.client.query(api.cryptoAnalysis.getTopSignals, { limit });
      } catch (error) {
        console.error('❌ Failed to fetch top signals:', error);
        throw error;
      }
    } else {
      // Development: Use local storage
      try {
        if (!this.localData.cryptoAnalysis || this.localData.cryptoAnalysis.length === 0) {
          return [];
        }

        // Get signals from all analyses
        const allSignals = this.localData.cryptoAnalysis
          .flatMap((analysis: any) => analysis.signals || [])
          .sort((a: any, b: any) => (b.score || 0) - (a.score || 0));

        return allSignals.slice(0, limit);
      } catch (error) {
        console.error('❌ Failed to fetch top signals from local database:', error);
        throw error;
      }
    }
  }

  // API Methods for local database
  async getTweets(): Promise<any> {
    if (this.client) {
      try {
        const api = await getConvexApi();
        return await this.client!.query(api.cryptoAnalysis.getTweets);
      } catch (error) {
        console.error('❌ Failed to fetch tweets:', error);
        throw error;
      }
    } else {
      return this.localData.tweets || [];
    }
  }

  async getAnalysis(): Promise<any> {
    if (this.client) {
      try {
        const api = await getConvexApi();
        return await this.client!.query(api.cryptoAnalysis.getAnalysis);
      } catch (error) {
        console.error('❌ Failed to fetch analysis:', error);
        throw error;
      }
    } else {
      return this.localData.cryptoAnalysis || [];
    }
  }

  async getAccounts(): Promise<any> {
    if (this.client) {
      try {
        const api = await getConvexApi();
        return await this.client!.query(api.cryptoAnalysis.getAccounts);
      } catch (error) {
        console.error('❌ Failed to fetch accounts:', error);
        throw error;
      }
    } else {
      return this.localData.accounts || [];
    }
  }

  // Get trending tokens
  async getTrendingTokens(limit = 10): Promise<any> {
    if (this.client) {
      try {
      const api = await getConvexApi();
      return await this.client!.query(api.cryptoAnalysis.getTrendingTokens, { limit });
      } catch (error) {
        console.error('❌ Failed to fetch trending tokens:', error);
        throw error;
      }
    } else {
      // Derive trending tokens from local cryptoAnalysis data
      const analyses = this.localData.cryptoAnalysis || [];
      const counts: Record<string, number> = {};
      for (const a of analyses) {
        for (const m of a.tokenMentions || []) {
          counts[m.symbol] = (counts[m.symbol] || 0) + m.count;
        }
      }
      return Object.entries(counts)
        .sort((a, b) => b[1] - a[1])
        .slice(0, limit)
        .map(([symbol, mentions]) => ({ symbol, mentions }));
    }
  }

  // Batch save token analyses
  async saveMultipleTokens(tokenDataArray: Array<{
    symbol: string;
    lastUpdated: string;
    totalMentions: number;
    sentimentScore: number;
    sentiment: string;
    botScore: number;
    shillProbability: number;
    avgEngagement: number;
    uniqueAccounts: number;
    recentTweets: string[];
    signals: any;
  }>): Promise<Array<{ success: boolean; symbol: string; id?: string; error?: unknown }>> {
    const results: Array<{ success: boolean; symbol: string; id?: string; error?: unknown }> = [];
    for (const tokenData of tokenDataArray) {
      try {
        const result = await this.saveTokenAnalysis(tokenData);
        results.push({ success: true, symbol: tokenData.symbol, id: result });
      } catch (error) {
        results.push({ success: false, symbol: tokenData.symbol, error });
      }
    }
    return results;
  }

  // Batch save account reputations
  async saveMultipleAccounts(accountDataArray: Array<{
    username: string;
    lastUpdated: string;
    tweetCount: number;
    tokensPosted: string[];
    avgEngagement: number;
    botScore: number;
    shillScore: number;
    reputation: 'trusted' | 'suspicious' | 'bot';
    totalMentions: number;
    followerCount?: number;
    accountAge?: number;
  }>): Promise<Array<{ success: boolean; username: string; id?: string; error?: unknown }>> {
    const results: Array<{ success: boolean; username: string; id?: string; error?: unknown }> = [];
    for (const accountData of accountDataArray) {
      try {
        const result = await this.saveAccountReputation(accountData);
        results.push({ success: true, username: accountData.username, id: result });
      } catch (error) {
        results.push({ success: false, username: accountData.username, error });
      }
    }
    return results;
  }
}

// Export singleton instance
export const convexStorage: ConvexStorage = new ConvexStorage();