ai-functions/test/blog-generation.test.js

Core AI primitives for building intelligent applications

/**
 * Blog Post Generation Live Tests
 *
 * These tests run LIVE against real AI providers by default.
 * They verify the complete blog generation workflow:
 *
 * ```ts
 * const titles = await list`10 blog post titles about ${topic}`
 * const posts = titles.map(title => write`a blog post starting with "# ${title}"`)
 * ```
 *
 * Tests cover:
 * - Real API calls to OpenAI, Anthropic, etc.
 * - Action/event storage in the database
 * - Both realtime and batch execution modes
 * - Multiple providers
 *
 * Run with:
 * ```bash
 * pnpm test blog-generation.live
 * ```
 *
 * Skip live tests (CI without API keys):
 * ```bash
 * SKIP_LIVE_TESTS=true pnpm test blog-generation.live
 * ```
 *
 * @packageDocumentation
 */
import { describe, it, expect, beforeEach, afterEach, beforeAll } from 'vitest';
import { configure, resetContext, withContext, getProvider, getModel, } from '../src/context.js';
import { createBatch } from '../src/batch-queue.js';
import { generateObject, generateText } from '../src/generate.js';
// Database provider for action/event storage
import { createMemoryProvider } from '../../ai-database/src/memory-provider.js';
// Batch storage
import '../src/batch/memory.js';
import { configureMemoryAdapter, clearBatches, getBatches } from '../src/batch/memory.js';
// ============================================================================
// Configuration
// ============================================================================
const SKIP_LIVE = process.env.SKIP_LIVE_TESTS === 'true';
const describeLive = SKIP_LIVE ? describe.skip : describe;
// Detect available providers
const hasOpenAI = !!process.env.OPENAI_API_KEY;
const hasAnthropic = !!process.env.ANTHROPIC_API_KEY;
// Provider configs
const PROVIDERS = {
    openai: { model: 'gpt-4o-mini', available: hasOpenAI },
    anthropic: { model: 'claude-sonnet-4-20250514', available: hasAnthropic },
};
// Get first available provider
const defaultProvider = Object.entries(PROVIDERS).find(([, cfg]) => cfg.available)?.[0] || 'openai';
const defaultModel = PROVIDERS[defaultProvider]?.model || 'gpt-4o-mini';
// ============================================================================
// Database Setup
// ============================================================================
let db;
let capturedEvents = [];
// ============================================================================
// Test Helpers
// ============================================================================
async function createAction(data) {
    return db.createAction({
        actor: 'test:live',
        action: data.action,
        object: data.object,
        objectData: data.objectData,
        total: data.total,
    });
}
async function generateTitles(topic, count) {
    const action = await createAction({
        action: 'generate',
        object: 'BlogTitles',
        objectData: { topic, count },
        total: 1,
    });
    await db.updateAction(action.id, { status: 'active' });
    const result = await generateObject({
        model: getModel(),
        schema: { titles: [`${count} blog post titles about ${topic}`] },
        prompt: `Generate exactly ${count} creative blog post titles about "${topic}".`,
    });
    const titles = result.object.titles;
    await db.updateAction(action.id, {
        status: 'completed',
        progress: 1,
        result: { titles },
    });
    return { titles, action };
}
async function generatePost(title) {
    const result = await generateText({
        model: getModel(),
        prompt: `Write a blog post starting with "# ${title}"\n\nInclude an introduction, 2-3 sections, and conclusion. Be concise.`,
        maxTokens: 800,
    });
    return result.text;
}
async function generatePosts(titles, mode = 'realtime') {
    const action = await createAction({
        action: 'generate',
        object: 'BlogPosts',
        objectData: { titles, mode },
        total: titles.length,
    });
    await db.updateAction(action.id, { status: 'active' });
    const posts = [];
    if (mode === 'batch') {
        const batch = createBatch({
            provider: getProvider(),
            model: getModel(),
            metadata: { actionId: action.id },
        });
        titles.forEach((title, i) => {
            batch.add(`Write a blog post starting with "# ${title}"\n\nBe concise.`, { customId: `post-${i}`, metadata: { title } });
        });
        const { completion } = await batch.submit();
        const results = await completion;
        for (const result of results) {
            posts.push(result.status === 'completed' ? result.result : `[Failed]`);
            await db.updateAction(action.id, { progress: posts.length });
        }
    }
    else {
        for (let i = 0; i < titles.length; i++) {
            const post = await generatePost(titles[i]);
            posts.push(post);
            await db.updateAction(action.id, { progress: i + 1 });
        }
    }
    await db.updateAction(action.id, { status: 'completed', result: { count: posts.length } });
    return { posts, action };
}
// ============================================================================
// Tests
// ============================================================================
describeLive('Blog Generation (Live)', () => {
    beforeAll(() => {
        console.log('\n🔴 LIVE TEST MODE');
        console.log(`   Default provider: ${defaultProvider}`);
        console.log(`   OpenAI available: ${hasOpenAI}`);
        console.log(`   Anthropic available: ${hasAnthropic}\n`);
    });
    beforeEach(() => {
        resetContext();
        db = createMemoryProvider();
        capturedEvents = [];
        db.on('*', (e) => capturedEvents.push(e));
        configure({ provider: defaultProvider, model: defaultModel, batchMode: 'immediate' });
        clearBatches();
        configureMemoryAdapter({});
    });
    afterEach(() => {
        resetContext();
        clearBatches();
        db.clear();
    });
    // ==========================================================================
    // Core Pattern Tests
    // ==========================================================================
    describe('Core Pattern: list → write', () => {
        it('generates titles from a topic', async () => {
            const { titles, action } = await generateTitles('building AI products', 3);
            expect(titles).toHaveLength(3);
            titles.forEach((t) => expect(typeof t).toBe('string'));
            // Verify action tracking
            expect(action.status).toBe('completed');
            expect(action.result).toEqual({ titles });
        }, 30000);
        it('generates a blog post from a title', async () => {
            const title = 'The Future of AI Development';
            const post = await generatePost(title);
            expect(post).toContain(`# ${title}`);
            expect(post.length).toBeGreaterThan(200);
        }, 30000);
        it('generates multiple posts from titles', async () => {
            const { titles } = await generateTitles('startup growth', 2);
            const { posts, action } = await generatePosts(titles);
            expect(posts).toHaveLength(2);
            posts.forEach((post, i) => {
                expect(post).toContain(`# ${titles[i]}`);
            });
            expect(action.progress).toBe(2);
            expect(action.total).toBe(2);
        }, 60000);
    });
    // ==========================================================================
    // Action & Event Storage
    // ==========================================================================
    describe('Action & Event Storage', () => {
        it('creates actions with verb conjugation', async () => {
            const { action } = await generateTitles('test topic', 2);
            expect(action.action).toBe('generate');
            expect(action.act).toBe('generates');
            expect(action.activity).toBe('generating');
            expect(action.actor).toBe('test:live');
        }, 30000);
        it('tracks action lifecycle timestamps', async () => {
            const { action } = await generateTitles('test', 2);
            const final = await db.getAction(action.id);
            expect(final?.createdAt).toBeInstanceOf(Date);
            expect(final?.startedAt).toBeInstanceOf(Date);
            expect(final?.completedAt).toBeInstanceOf(Date);
            expect(final?.startedAt.getTime()).toBeGreaterThanOrEqual(final?.createdAt.getTime());
            expect(final?.completedAt.getTime()).toBeGreaterThanOrEqual(final?.startedAt.getTime());
        }, 30000);
        it('emits events for state transitions', async () => {
            await generateTitles('test', 2);
            const actionEvents = capturedEvents.filter((e) => e.event.startsWith('Action.'));
            const eventTypes = actionEvents.map((e) => e.event);
            expect(eventTypes).toContain('Action.created');
            expect(eventTypes).toContain('Action.started');
            expect(eventTypes).toContain('Action.completed');
        }, 30000);
        it('stores action result data', async () => {
            const { titles, action } = await generateTitles('AI development', 3);
            const stored = await db.getAction(action.id);
            expect(stored?.result).toEqual({ titles });
            expect(stored?.objectData).toEqual({ topic: 'AI development', count: 3 });
        }, 30000);
        it('queries actions by status', async () => {
            await generateTitles('topic 1', 2);
            await generateTitles('topic 2', 2);
            const completed = await db.listActions({ status: 'completed' });
            expect(completed.length).toBe(2);
        }, 60000);
        it('queries events by type pattern', async () => {
            await generateTitles('test', 2);
            const created = await db.listEvents({ event: 'Action.created' });
            const allAction = await db.listEvents({ event: 'Action.*' });
            expect(created.length).toBeGreaterThanOrEqual(1);
            expect(allAction.length).toBeGreaterThanOrEqual(3); // created, started, completed
        }, 30000);
    });
    // ==========================================================================
    // Realtime Execution
    // ==========================================================================
    describe('Realtime Execution', () => {
        beforeEach(() => {
            configure({ batchMode: 'immediate' });
        });
        it('executes requests immediately', async () => {
            const start = Date.now();
            const { titles } = await generateTitles('quick test', 2);
            const elapsed = Date.now() - start;
            expect(titles).toHaveLength(2);
            // Should complete in reasonable time (not batched/delayed)
            expect(elapsed).toBeLessThan(30000);
        }, 30000);
        it('tracks progress during sequential generation', async () => {
            const titles = ['Post One', 'Post Two'];
            const { posts, action } = await generatePosts(titles, 'realtime');
            expect(posts).toHaveLength(2);
            const final = await db.getAction(action.id);
            expect(final?.progress).toBe(2);
            expect(final?.total).toBe(2);
        }, 60000);
    });
    // ==========================================================================
    // Batch Execution
    // ==========================================================================
    describe('Batch Execution', () => {
        beforeEach(() => {
            configure({ batchMode: 'deferred' });
        });
        it('creates and submits batch jobs', async () => {
            const titles = ['Batch Post 1', 'Batch Post 2'];
            const { posts, action } = await generatePosts(titles, 'batch');
            expect(posts).toHaveLength(2);
            // Verify batch was stored
            const batches = getBatches();
            expect(batches.size).toBe(1);
            const [, batch] = [...batches.entries()][0];
            expect(batch.items).toHaveLength(2);
            expect(batch.status).toBe('completed');
        }, 90000);
        it('stores batch metadata', async () => {
            const batch = createBatch({
                provider: getProvider(),
                model: getModel(),
                metadata: { task: 'test-batch', timestamp: Date.now() },
            });
            batch.add('Write a test post', { customId: 'test-1' });
            const { job } = await batch.submit();
            await batch.wait();
            const stored = getBatches().get(job.id);
            expect(stored?.options.metadata?.task).toBe('test-batch');
        }, 30000);
    });
    // ==========================================================================
    // Multi-Provider Tests
    // ==========================================================================
    describe('Multi-Provider', () => {
        it.skipIf(!hasOpenAI)('generates with OpenAI', async () => {
            configure({ provider: 'openai', model: 'gpt-4o-mini' });
            const { titles } = await generateTitles('OpenAI test', 2);
            expect(titles).toHaveLength(2);
            expect(getProvider()).toBe('openai');
        }, 30000);
        it.skipIf(!hasAnthropic)('generates with Anthropic', async () => {
            configure({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' });
            const { titles } = await generateTitles('Anthropic test', 2);
            expect(titles).toHaveLength(2);
            expect(getProvider()).toBe('anthropic');
        }, 30000);
        it.skipIf(!hasOpenAI || !hasAnthropic)('switches providers mid-workflow', async () => {
            // Generate titles with OpenAI
            configure({ provider: 'openai', model: 'gpt-4o-mini' });
            const { titles } = await generateTitles('cross-provider test', 2);
            // Generate posts with Anthropic
            const posts = await withContext({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' }, async () => {
                expect(getProvider()).toBe('anthropic');
                return Promise.all(titles.slice(0, 1).map(generatePost));
            });
            expect(posts).toHaveLength(1);
            expect(getProvider()).toBe('openai'); // restored
        }, 60000);
        it.skipIf(!hasOpenAI || !hasAnthropic)('runs providers in parallel', async () => {
            const [openaiResult, anthropicResult] = await Promise.all([
                withContext({ provider: 'openai', model: 'gpt-4o-mini' }, () => generateTitles('OpenAI parallel', 2)),
                withContext({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' }, () => generateTitles('Anthropic parallel', 2)),
            ]);
            expect(openaiResult.titles).toHaveLength(2);
            expect(anthropicResult.titles).toHaveLength(2);
        }, 60000);
    });
    // ==========================================================================
    // Error Handling
    // ==========================================================================
    describe('Error Handling', () => {
        it('tracks failed actions', async () => {
            const action = await createAction({
                action: 'generate',
                object: 'FailTest',
                total: 1,
            });
            await db.updateAction(action.id, { status: 'active' });
            // Simulate failure
            await db.updateAction(action.id, {
                status: 'failed',
                error: 'Simulated failure',
            });
            const failed = await db.listActions({ status: 'failed' });
            expect(failed).toHaveLength(1);
            expect(failed[0].error).toBe('Simulated failure');
            // Verify failure event
            const failEvents = capturedEvents.filter((e) => e.event === 'Action.failed');
            expect(failEvents.length).toBe(1);
        });
        it('handles invalid model gracefully', async () => {
            configure({ model: 'invalid-model-xyz' });
            const action = await createAction({
                action: 'generate',
                object: 'InvalidModelTest',
                total: 1,
            });
            await db.updateAction(action.id, { status: 'active' });
            try {
                await generateObject({
                    model: 'invalid-model-xyz',
                    schema: { test: 'test' },
                    prompt: 'test',
                });
            }
            catch (e) {
                await db.updateAction(action.id, {
                    status: 'failed',
                    error: e instanceof Error ? e.message : 'Unknown error',
                });
            }
            const final = await db.getAction(action.id);
            expect(final?.status).toBe('failed');
            expect(final?.error).toBeDefined();
        }, 30000);
    });
    // ==========================================================================
    // Database Statistics
    // ==========================================================================
    describe('Database Statistics', () => {
        it('tracks aggregate stats', async () => {
            await generateTitles('stats test 1', 2);
            await generateTitles('stats test 2', 2);
            const stats = db.stats();
            expect(stats.actions.completed).toBe(2);
            expect(stats.events).toBeGreaterThan(0);
        }, 60000);
    });
});