@podx/core/README.md

🚀 Core utilities and shared functionality for PODx - Advanced Twitter/X scraping and crypto analysis toolkit

# @podx/core

[![Version](https://img.shields.io/badge/version-2.0.0-blue.svg)](https://github.com/podx/core)
[![License](https://img.shields.io/badge/license-ISC-green.svg)](https://github.com/podx/core/blob/main/LICENSE)
[![TypeScript](https://img.shields.io/badge/TypeScript-5.9+-blue.svg)](https://www.typescriptlang.org/)
[![Bun](https://img.shields.io/badge/Bun-1.0+-yellow.svg)](https://bun.sh/)

The core package provides essential utilities, types, configuration management, logging, error handling, and database operations for the PODx ecosystem. It serves as the foundational layer that other packages build upon.

## 📦 Installation

```bash
# Install from workspace
bun add @podx/core@workspace:*

# Or install from npm (when published)
bun add @podx/core
```

## 🏗️ Architecture

The core package is organized into several key modules:

```
packages/core/src/
├── config/           # Configuration management
├── convex/           # Convex database integration
├── errors/           # Error types and handling
├── logger/           # Structured logging
├── storage/          # File storage utilities
├── types/            # TypeScript type definitions
├── utils/            # General utilities
└── index.ts          # Main exports
```

## 🚀 Quick Start

```typescript
import { config, logger, DatabaseService } from '@podx/core';

// Load configuration
const appConfig = config.load();

// Initialize logger
const log = logger.createLogger('my-service');

// Use database service
const db = new DatabaseService(appConfig.database);
```

## 📚 Core Modules

### Configuration Management

```typescript
import { config } from '@podx/core';

// Load configuration from environment
const appConfig = config.load();

// Access configuration values
console.log(appConfig.database.url);
console.log(appConfig.twitter.apiKey);

// Validate configuration
const validation = config.validate(appConfig);
if (!validation.isValid) {
  console.error('Configuration errors:', validation.errors);
}
```

**Configuration Schema:**
```typescript
interface AppConfig {
  database: {
    url: string;
    maxConnections: number;
    timeout: number;
  };
  twitter: {
    apiKey: string;
    apiSecret: string;
    accessToken?: string;
    accessTokenSecret?: string;
  };
  logging: {
    level: 'debug' | 'info' | 'warn' | 'error';
    format: 'json' | 'text';
  };
  storage: {
    type: 'local' | 's3' | 'convex';
    localPath?: string;
    s3Bucket?: string;
  };
}
```

### Structured Logging with Pino

```typescript
import { logger, createModuleLogger, createServiceLogger } from '@podx/core';

// Use the main logger instance
logger.info('Application started');
logger.debug('Processing user data', { userId: '123', action: 'update' });
logger.warn('Rate limit approaching', { remaining: 10 });
logger.error('Failed to update user', { userId: '123', error: 'Database timeout' });

// Create module-specific loggers
const scraperLogger = createModuleLogger('scraper');
scraperLogger.info('Scraper initialized', { maxConcurrency: 5 });

// Create service loggers with version info
const apiLogger = createServiceLogger('api-server', '2.0.0');
apiLogger.info('API server started', { port: 3000, environment: 'production' });

// Specialized logging methods
logger.scrapeStart('username', 100);
logger.scrapeComplete('username', 85, 1250);
logger.apiRequest('GET', '/api/users', 200, 45);
logger.authFailure('invalid_token', { userId: '123' });

// Child loggers for request tracking
const requestLogger = logger.child({ requestId: 'abc-123', userId: 'user-456' });
requestLogger.info('Processing payment', { amount: 99.99, currency: 'USD' });
```

**Log Output (JSON):**
```json
{
  "level": 30,
  "time": 1705312245123,
  "pid": 12345,
  "hostname": "podx-server-01",
  "timestamp": "2025-01-15T10:30:45.123Z",
  "module": "scraper",
  "msg": "Scraper initialized",
  "maxConcurrency": 5
}
```

**Log Output (Pretty - Development):**
```
[2025-01-15 10:30:45.123] INFO  🔍 Starting scrape operation for @username | operation=scrape_start user=username maxTweets=100
[2025-01-15 10:30:45.124] INFO  🌐 GET /api/users | operation=api_request method=GET path=/api/users statusCode=200 duration=45
```

### Error Handling

```typescript
import { errors, Result } from '@podx/core';

// Define custom error types
class ValidationError extends errors.BaseError {
  readonly code = 'VALIDATION_ERROR';
  readonly statusCode = 400;

  constructor(field: string, value: unknown) {
    super(`Invalid value for field '${field}': ${value}`);
    this.context = { field, value };
  }
}

// Use Result pattern for operations
function validateUserData(data: unknown): Result<User, ValidationError> {
  try {
    const user = userSchema.parse(data);
    return { success: true, data: user };
  } catch (error) {
    return {
      success: false,
      error: new ValidationError('user', data)
    };
  }
}

// Handle results
const result = validateUserData(inputData);
if (result.success) {
  console.log('Valid user:', result.data);
} else {
  console.error('Validation failed:', result.error.message);
  // Error is properly typed and contains context
}
```

### Database Operations

```typescript
import { DatabaseService, convex } from '@podx/core';

// Initialize database service
const db = new DatabaseService(config.database);

// Convex integration
const convexClient = convex.createClient(config.convex);

// Define a data model
interface Tweet {
  id: string;
  content: string;
  authorId: string;
  createdAt: Date;
  metrics: {
    likes: number;
    retweets: number;
    replies: number;
  };
}

// Database operations
async function saveTweet(tweet: Tweet): Promise<Result<Tweet, DatabaseError>> {
  try {
    const savedTweet = await db.save('tweets', tweet);
    return { success: true, data: savedTweet };
  } catch (error) {
    return {
      success: false,
      error: new DatabaseError('Failed to save tweet', error)
    };
  }
}

// Query operations
async function getTweetsByAuthor(authorId: string): Promise<Tweet[]> {
  return db.query('tweets')
    .where('authorId', '==', authorId)
    .orderBy('createdAt', 'desc')
    .limit(50)
    .get();
}
```

### File Storage

```typescript
import { storage } from '@podx/core';

// Initialize storage
const store = storage.createStorage(config.storage);

// Save data
const data = { message: 'Hello World', timestamp: new Date() };
const key = await store.save('messages/hello.json', JSON.stringify(data));

// Load data
const loadedData = await store.load('messages/hello.json');
const parsed = JSON.parse(loadedData);

// List files
const files = await store.list('messages/');
console.log('Available messages:', files);

// Delete data
await store.delete('messages/hello.json');
```

## 🔧 Advanced Usage

### Custom Configuration

```typescript
import { config } from '@podx/core';

// Extend default configuration
const customConfig = config.extend({
  customFeature: {
    enabled: true,
    apiUrl: 'https://api.example.com',
    retryAttempts: 3
  }
});

// Use custom configuration
const app = new Application(customConfig);
```

### Pino Performance & Benchmarking

Pino is one of the fastest logging libraries for Node.js, optimized for high-performance applications.

```typescript
import { benchmarkLogger, logBenchmarkResults } from '@podx/core';

// Run performance benchmarks
await logBenchmarkResults(10000);

// Expected output:
// Benchmark result - info_logging: 1600 ops/sec, 0.625ms avg
// Benchmark result - child_logger: 633549 ops/sec, 0.0016ms avg
// Benchmark result - specialized_methods: 4474 ops/sec, 0.2235ms avg
```

**Performance Characteristics:**
- **Child Loggers**: ~630,000 ops/sec (extremely fast for request tracing)
- **Basic Logging**: ~1,600 ops/sec (excellent for general logging)
- **Specialized Methods**: ~4,500 ops/sec (great for API monitoring)
- **Memory Efficient**: Low memory footprint with streaming architecture
- **Async Safe**: Non-blocking I/O operations

### Advanced Logger Usage

```typescript
import { logger, createRequestLogger } from '@podx/core';

// Request-scoped logging
class RequestHandler {
  async handle(request: Request): Promise<Response> {
    const requestLogger = createRequestLogger(request.id, request.userId);

    requestLogger.info('Processing request', {
      method: request.method,
      path: request.path,
      userAgent: request.headers['user-agent']
    });

    try {
      const result = await this.processRequest(request);
      requestLogger.info('Request completed successfully', {
        statusCode: 200,
        duration: Date.now() - request.startTime
      });
      return result;
    } catch (error) {
      requestLogger.error('Request failed', {
        error: error.message,
        statusCode: 500,
        duration: Date.now() - request.startTime
      }, error);
      throw error;
    }
  }
}
```

### Custom Error Types

```typescript
import { errors } from '@podx/core';

// Create domain-specific errors
class TwitterAPIError extends errors.BaseError {
  readonly code = 'TWITTER_API_ERROR';
  readonly statusCode = 502;

  constructor(endpoint: string, statusCode: number, response: string) {
    super(`Twitter API request failed for ${endpoint}`);
    this.context = { endpoint, statusCode, response };
  }
}

class RateLimitError extends errors.BaseError {
  readonly code = 'RATE_LIMIT_ERROR';
  readonly statusCode = 429;

  constructor(endpoint: string, resetTime: Date) {
    super(`Rate limit exceeded for ${endpoint}`);
    this.context = { endpoint, resetTime };
  }
}

// Use in API client
class TwitterClient {
  async makeRequest(endpoint: string): Promise<Result<TwitterData, TwitterAPIError>> {
    try {
      const response = await this.httpClient.get(endpoint);

      if (response.status === 429) {
        const resetTime = new Date(response.headers['x-rate-limit-reset'] * 1000);
        return {
          success: false,
          error: new RateLimitError(endpoint, resetTime)
        };
      }

      if (!response.ok) {
        return {
          success: false,
          error: new TwitterAPIError(endpoint, response.status, response.data)
        };
      }

      return { success: true, data: response.data };
    } catch (error) {
      return {
        success: false,
        error: new TwitterAPIError(endpoint, 0, error.message)
      };
    }
  }
}
```

## 🔌 Integration Examples

### With Express.js

```typescript
import express from 'express';
import { logger, errors, config } from '@podx/core';

const app = express();
const log = logger.createLogger('api-server');
const cfg = config.load();

// Middleware
app.use(express.json());

// Request logging
app.use((req, res, next) => {
  const start = Date.now();
  log.info('Request started', {
    method: req.method,
    url: req.url,
    userAgent: req.get('User-Agent')
  });

  res.on('finish', () => {
    log.info('Request completed', {
      method: req.method,
      url: req.url,
      statusCode: res.statusCode,
      duration: Date.now() - start
    });
  });

  next();
});

// Error handling
app.use((error: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {
  if (error instanceof errors.BaseError) {
    log.error('Application error', {
      code: error.code,
      statusCode: error.statusCode,
      context: error.context,
      stack: error.stack
    });

    res.status(error.statusCode).json({
      error: {
        code: error.code,
        message: error.message,
        ...(cfg.node_env === 'development' && { context: error.context })
      }
    });
  } else {
    log.error('Unexpected error', { error: error.message, stack: error.stack });
    res.status(500).json({
      error: {
        code: 'INTERNAL_ERROR',
        message: 'An unexpected error occurred'
      }
    });
  }
});
```

### With Convex

```typescript
import { convex } from '@podx/core';

// Initialize Convex client
const client = convex.createClient({
  url: process.env.CONVEX_URL!,
  auth: {
    type: 'bearer',
    token: process.env.CONVEX_ACCESS_TOKEN!
  }
});

// Define mutation
export const saveTweet = convex.mutation(async ({ db }, tweet: Tweet) => {
  const log = logger.createLogger('convex-mutation');

  try {
    const result = await db.insert('tweets', {
      ...tweet,
      createdAt: new Date().toISOString(),
      updatedAt: new Date().toISOString()
    });

    log.info('Tweet saved', { tweetId: result.id });
    return result;
  } catch (error) {
    log.error('Failed to save tweet', { error: error.message });
    throw error;
  }
});

// Define query
export const getTweets = convex.query(async ({ db }, authorId: string) => {
  const log = logger.createLogger('convex-query');

  try {
    const tweets = await db
      .query('tweets')
      .withIndex('by_author', q => q.eq('authorId', authorId))
      .order('desc')
      .take(50);

    log.info('Tweets retrieved', { authorId, count: tweets.length });
    return tweets;
  } catch (error) {
    log.error('Failed to get tweets', { authorId, error: error.message });
    throw error;
  }
});
```

## 📊 API Reference

### Configuration

#### `config.load(): AppConfig`
Loads configuration from environment variables and validates the schema.

#### `config.validate(config: AppConfig): ValidationResult`
Validates a configuration object against the expected schema.

#### `config.extend<T>(extension: T): AppConfig & T`
Extends the default configuration with custom properties.

### Logging with Pino

#### Main Logger Instance
```typescript
import { logger } from '@podx/core';

// Direct usage
logger.info('Application started');
logger.error('Database error', { error: 'Connection failed' });
```

#### Logger Methods
```typescript
interface Logger {
  debug(message: string, context?: LogContext): void;
  info(message: string, context?: LogContext): void;
  warn(message: string, context?: LogContext): void;
  error(message: string, context?: LogContext, error?: Error): void;
  trace(message: string, context?: LogContext): void;

  // Specialized methods
  scrapeStart(username: string, maxTweets: number): void;
  scrapeComplete(username: string, tweetCount: number, duration: number): void;
  apiRequest(method: string, path: string, statusCode?: number, duration?: number): void;
  authFailure(reason: string, context?: LogContext): void;
  rateLimitExceeded(identifier: string, limit: number): void;

  // Child loggers
  child(bindings: Record<string, any>): Logger;
}
```

#### Logger Factory Functions
```typescript
import {
  createModuleLogger,
  createServiceLogger,
  createRequestLogger,
  createScraperLogger,
  createApiLogger
} from '@podx/core';

const moduleLogger = createModuleLogger('scraper');
const serviceLogger = createServiceLogger('api', '2.0.0');
const requestLogger = createRequestLogger('req-123', 'user-456');
const scraperLogger = createScraperLogger('twitter-scraper', '@targetUser');
const apiLogger = createApiLogger('GET', '/api/users');
```

#### Configuration
```typescript
import { configureLogger, setLogLevel, setLogFile, enablePrettyLogging } from '@podx/core';

// Configure via environment variables or programmatically
configureLogger({
  level: LogLevel.DEBUG,
  logFile: '/var/log/podx/app.log',
  enableConsole: true,
  enableFile: true
});

// Utility functions
setLogLevel(LogLevel.INFO);
setLogFile('/var/log/podx/app.log');
enablePrettyLogging();
```

### Errors

#### `errors.BaseError`
Base error class with context and status code support.

```typescript
class BaseError extends Error {
  readonly code: string;
  readonly statusCode: number;
  readonly timestamp: number;
  readonly context?: Record<string, unknown>;
}
```

#### `Result<T, E>`
Result type for error handling.

```typescript
type Result<T, E = Error> =
  | { success: true; data: T }
  | { success: false; error: E };
```

### Database

#### `DatabaseService`
Service for database operations.

```typescript
class DatabaseService {
  constructor(config: DatabaseConfig);

  save<T>(collection: string, data: T): Promise<T>;
  findById<T>(collection: string, id: string): Promise<T | null>;
  query<T>(collection: string): QueryBuilder<T>;
  delete(collection: string, id: string): Promise<void>;
  update<T>(collection: string, id: string, data: Partial<T>): Promise<T>;
}
```

### Storage

#### `StorageService`
Abstract interface for file storage operations.

```typescript
interface StorageService {
  save(key: string, data: string | Buffer): Promise<string>;
  load(key: string): Promise<string>;
  delete(key: string): Promise<void>;
  list(prefix?: string): Promise<string[]>;
  exists(key: string): Promise<boolean>;
}
```

## 🔍 Type Definitions

### Core Types

```typescript
// Configuration
export interface AppConfig {
  database: DatabaseConfig;
  twitter: TwitterConfig;
  logging: LoggingConfig;
  storage: StorageConfig;
}

// Database
export interface DatabaseConfig {
  url: string;
  maxConnections: number;
  timeout: number;
  ssl?: boolean;
}

// Twitter API
export interface TwitterConfig {
  apiKey: string;
  apiSecret: string;
  accessToken?: string;
  accessTokenSecret?: string;
  bearerToken?: string;
}

// Logging
export interface LoggingConfig {
  level: 'debug' | 'info' | 'warn' | 'error';
  format: 'json' | 'text';
  outputs: ('console' | 'file')[];
  filePath?: string;
}

// Storage
export interface StorageConfig {
  type: 'local' | 's3' | 'convex';
  localPath?: string;
  s3Bucket?: string;
  s3Region?: string;
}
```

## 🧪 Testing

```typescript
import { describe, test, expect, mock } from 'bun:test';
import { config, logger } from '@podx/core';

describe('Configuration', () => {
  test('should load configuration from environment', () => {
    // Mock environment variables
    process.env.DATABASE_URL = 'postgresql://localhost:5432/podx';
    process.env.TWITTER_API_KEY = 'test-key';

    const appConfig = config.load();

    expect(appConfig.database.url).toBe('postgresql://localhost:5432/podx');
    expect(appConfig.twitter.apiKey).toBe('test-key');
  });

  test('should validate configuration', () => {
    const invalidConfig = { database: {} };
    const validation = config.validate(invalidConfig);

    expect(validation.isValid).toBe(false);
    expect(validation.errors).toContain('database.url is required');
  });
});

describe('Logger', () => {
  test('should create logger with correct name', () => {
    const log = logger.createLogger('test-service');

    expect(log).toBeDefined();
    expect(typeof log.info).toBe('function');
  });

  test('should support child loggers', () => {
    const parentLog = logger.createLogger('parent');
    const childLog = parentLog.child('child');

    expect(childLog).toBeDefined();
    expect(typeof childLog.debug).toBe('function');
  });
});
```

## 🤝 Contributing

1. Fork the repository
2. Create a feature branch
3. Make your changes
4. Add tests for new functionality
5. Ensure all tests pass
6. Submit a pull request

## 📝 License

This package is licensed under the ISC License. See the [LICENSE](LICENSE) file for details.

## 🔗 Related Packages

- [@podx/scraper](../scraper/README.md) - Twitter scraping functionality
- [@podx/api](../api/README.md) - REST API server
- [@podx/cli](../cli/README.md) - Command-line interface
- [podx](../podx/README.md) - Main CLI application

## 📞 Support

For support and questions:
- 📧 Email: support@podx.dev
- 💬 Discord: [PODx Community](https://discord.gg/podx)
- 📖 Documentation: [docs.podx.dev](https://docs.podx.dev)
- 🐛 Issues: [GitHub Issues](https://github.com/podx/core/issues)