@llumiverse/drivers/src/openai/error-handling.test.ts

LLM driver implementations. Currently supported are: openai, huggingface, bedrock, replicate.

import { describe, it, expect, beforeEach } from 'vitest';
import { BaseOpenAIDriver } from './index.js';
import { LlumiverseError, LlumiverseErrorContext, Providers } from '@llumiverse/core';
import OpenAI from 'openai';
import {
    APIError,
    BadRequestError,
    AuthenticationError,
    PermissionDeniedError,
    NotFoundError,
    ConflictError,
    UnprocessableEntityError,
    RateLimitError,
    InternalServerError,
    APIConnectionError,
    APIConnectionTimeoutError,
    LengthFinishReasonError,
    ContentFilterFinishReasonError,
    OpenAIError,
} from 'openai/error';

// Test implementation of BaseOpenAIDriver
class TestOpenAIDriver extends BaseOpenAIDriver {
    provider: Providers.openai = Providers.openai;
    service: OpenAI;

    constructor() {
        super({});
        this.service = new OpenAI({ apiKey: 'test-key' });
    }
}

describe('BaseOpenAIDriver Error Handling', () => {
    let driver: TestOpenAIDriver;

    beforeEach(() => {
        driver = new TestOpenAIDriver();
    });

    describe('formatLlumiverseError', () => {
        it('should handle BadRequestError with status code and parameter', () => {
            const headers = new Headers();
            headers.set('x-request-id', 'req_test_123');
            
            const openaiError = new BadRequestError(
                400,
                {
                    message: "Invalid 'temperature': decimal above maximum value. Expected a value <= 2, but got 31313 instead.",
                    type: 'invalid_request_error',
                    param: 'temperature',
                    code: 'decimal_above_max_value'
                },
                "Invalid 'temperature': decimal above maximum value. Expected a value <= 2, but got 31313 instead.",
                headers
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error).toBeInstanceOf(LlumiverseError);
            expect(error.code).toBe(400);
            expect(error.message).toContain('[400]');
            expect(error.message).toContain('temperature');
            expect(error.message).toContain('decimal_above_max_value');
            expect(error.message).toContain('req_test_123');
            expect(error.name).toBe('BadRequestError');
            expect(error.retryable).toBe(false);
        });

        it('should handle RateLimitError as retryable', () => {
            const headers = new Headers();
            const openaiError = new RateLimitError(
                429,
                {
                    message: 'Rate limit exceeded',
                    type: 'rate_limit_error',
                    code: 'rate_limit_exceeded'
                },
                'Rate limit exceeded',
                headers
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBe(429);
            expect(error.retryable).toBe(true);
            expect(error.name).toBe('RateLimitError');
        });

        it('should handle InternalServerError as retryable', () => {
            const openaiError = new InternalServerError(
                500,
                {
                    message: 'Internal server error',
                    type: 'server_error',
                    code: 'server_error'
                },
                'Internal server error',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBe(500);
            expect(error.retryable).toBe(true);
            expect(error.name).toBe('InternalServerError');
        });

        it('should handle AuthenticationError as not retryable', () => {
            const openaiError = new AuthenticationError(
                401,
                {
                    message: 'Invalid API key',
                    type: 'authentication_error',
                    code: 'invalid_api_key'
                },
                'Invalid API key',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBe(401);
            expect(error.retryable).toBe(false);
            expect(error.name).toBe('AuthenticationError');
        });

        it('should handle PermissionDeniedError as not retryable', () => {
            const openaiError = new PermissionDeniedError(
                403,
                {
                    message: 'Insufficient permissions',
                    type: 'permission_error'
                },
                'Insufficient permissions',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBe(403);
            expect(error.retryable).toBe(false);
            expect(error.name).toBe('PermissionDeniedError');
        });

        it('should handle NotFoundError as not retryable', () => {
            const openaiError = new NotFoundError(
                404,
                {
                    message: 'Model not found',
                    type: 'invalid_request_error',
                    code: 'model_not_found'
                },
                'Model not found',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-invalid',
                operation: 'execute',
            });

            expect(error.code).toBe(404);
            expect(error.retryable).toBe(false);
            expect(error.name).toBe('NotFoundError');
        });

        it('should handle ConflictError as not retryable', () => {
            const openaiError = new ConflictError(
                409,
                {
                    message: 'Resource conflict',
                    type: 'conflict_error'
                },
                'Resource conflict',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBe(409);
            expect(error.retryable).toBe(false);
            expect(error.name).toBe('ConflictError');
        });

        it('should handle UnprocessableEntityError as not retryable', () => {
            const openaiError = new UnprocessableEntityError(
                422,
                {
                    message: 'Validation failed',
                    type: 'validation_error'
                },
                'Validation failed',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBe(422);
            expect(error.retryable).toBe(false);
            expect(error.name).toBe('UnprocessableEntityError');
        });

        it('should handle APIConnectionTimeoutError as retryable', () => {
            const openaiError = new APIConnectionTimeoutError({ message: 'Request timed out' });

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBeUndefined();
            expect(error.retryable).toBe(true);
            expect(error.name).toBe('APIConnectionTimeoutError');
        });

        it('should handle APIConnectionError as retryable', () => {
            const openaiError = new APIConnectionError({ message: 'Connection failed' });

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.code).toBeUndefined();
            expect(error.retryable).toBe(true);
            expect(error.name).toBe('APIConnectionError');
        });

        it('should handle LengthFinishReasonError as not retryable', () => {
            const openaiError = new LengthFinishReasonError();

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.retryable).toBe(false);
            expect(error.name).toBe('LengthFinishReasonError');
        });

        it('should handle ContentFilterFinishReasonError as not retryable', () => {
            const openaiError = new ContentFilterFinishReasonError();

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.retryable).toBe(false);
            expect(error.name).toBe('ContentFilterFinishReasonError');
        });

        it('should include error code in message', () => {
            const headers = new Headers();
            const openaiError = new BadRequestError(
                400,
                {
                    message: 'Invalid parameter',
                    code: 'invalid_parameter'
                },
                'Invalid parameter',
                headers
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.message).toContain('invalid_parameter');
        });

        it('should include parameter in message when available', () => {
            const headers = new Headers();
            const openaiError = new BadRequestError(
                400,
                {
                    message: 'Invalid value',
                    param: 'max_tokens',
                    code: 'invalid_value'
                },
                'Invalid value',
                headers
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.message).toContain('max_tokens');
        });

        it('should include request ID when available', () => {
            const headers = new Headers();
            headers.set('x-request-id', 'req_xyz789');
            
            const openaiError = new BadRequestError(
                400,
                { message: 'Bad request' },
                'Bad request',
                headers
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.message).toContain('req_xyz789');
        });

        it('should throw for non-OpenAI errors', () => {
            const regularError = new Error('Regular error');

            expect(() => {
                driver.formatLlumiverseError( regularError, {
                    provider: 'openai',
                    model: 'gpt-4',
                    operation: 'execute',
                });
            }).toThrow('Regular error');
        });

        it('should preserve original error for debugging', () => {
            const openaiError = new BadRequestError(
                400,
                { message: 'Test error' },
                'Test error',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error.originalError).toBe(openaiError);
        });
    });

    describe('isOpenAIErrorRetryable', () => {
        it('should classify retryable error types correctly', () => {
            const retryableErrors = [
                new RateLimitError(429, {}, 'Rate limit', new Headers()),
                new InternalServerError(500, {}, 'Server error', new Headers()),
                new APIConnectionTimeoutError({ message: 'Timeout' }),
            ];

            for (const error of retryableErrors) {
                const result = (driver as any).isOpenAIErrorRetryable(error, error.status, null, undefined);
                expect(result, `${error.constructor.name} should be retryable`).toBe(true);
            }
        });

        it('should classify non-retryable error types correctly', () => {
            const nonRetryableErrors = [
                new BadRequestError(400, {}, 'Bad request', new Headers()),
                new AuthenticationError(401, {}, 'Auth error', new Headers()),
                new PermissionDeniedError(403, {}, 'Permission denied', new Headers()),
                new NotFoundError(404, {}, 'Not found', new Headers()),
                new ConflictError(409, {}, 'Conflict', new Headers()),
                new UnprocessableEntityError(422, {}, 'Validation error', new Headers()),
                new LengthFinishReasonError(),
                new ContentFilterFinishReasonError(),
            ];

            for (const error of nonRetryableErrors) {
                const status = 'status' in error ? (error as any).status : undefined;
                const result = (driver as any).isOpenAIErrorRetryable(error, status, null, undefined);
                expect(result, `${error.constructor.name} should not be retryable`).toBe(false);
            }
        });

        it('should classify retryable error codes correctly', () => {
            const apiError = new APIError(undefined as any, {}, 'Error', new Headers());
            
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'timeout', undefined)).toBe(true);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'server_error', undefined)).toBe(true);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'service_unavailable', undefined)).toBe(true);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'rate_limit_exceeded', undefined)).toBe(true);
        });

        it('should classify non-retryable error codes correctly', () => {
            const apiError = new APIError(undefined as any, {}, 'Error', new Headers());
            
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'invalid_api_key', undefined)).toBe(false);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'invalid_request_error', undefined)).toBe(false);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'model_not_found', undefined)).toBe(false);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'insufficient_quota', undefined)).toBe(false);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'invalid_model', undefined)).toBe(false);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, 'invalid_parameter', undefined)).toBe(false);
        });

        it('should classify error types correctly', () => {
            const apiError = new APIError(undefined as any, {}, 'Error', new Headers());
            
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, null, 'invalid_request_error')).toBe(false);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, null, 'authentication_error')).toBe(false);
        });

        it('should use HTTP status codes when available', () => {
            const apiError = new APIError(429, {}, 'Too many requests', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError, 429, null, undefined)).toBe(true);
            
            const apiError2 = new APIError(408, {}, 'Request timeout', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError2, 408, null, undefined)).toBe(true);
            
            const apiError3 = new APIError(502, {}, 'Bad gateway', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError3, 502, null, undefined)).toBe(true);
            
            const apiError4 = new APIError(503, {}, 'Service unavailable', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError4, 503, null, undefined)).toBe(true);
            
            const apiError5 = new APIError(504, {}, 'Gateway timeout', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError5, 504, null, undefined)).toBe(true);
            
            const apiError6 = new APIError(529, {}, 'Overloaded', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError6, 529, null, undefined)).toBe(true);
        });

        it('should classify 4xx as non-retryable', () => {
            const apiError = new APIError(400, {}, 'Bad request', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError, 400, null, undefined)).toBe(false);
            
            const apiError2 = new APIError(403, {}, 'Forbidden', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError2, 403, null, undefined)).toBe(false);
        });

        it('should classify 5xx as retryable', () => {
            const apiError = new APIError(500, {}, 'Internal error', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError, 500, null, undefined)).toBe(true);
            
            const apiError2 = new APIError(502, {}, 'Bad gateway', new Headers());
            expect((driver as any).isOpenAIErrorRetryable(apiError2, 502, null, undefined)).toBe(true);
        });

        it('should classify APIConnectionError (non-timeout) as retryable', () => {
            const connectionError = new APIConnectionError({ message: 'Network failure' });
            expect((driver as any).isOpenAIErrorRetryable(connectionError, undefined, null, undefined)).toBe(true);
        });

        it('should return undefined for unknown errors', () => {
            const apiError = new APIError(undefined as any, {}, 'Unknown error', undefined as any);
            expect((driver as any).isOpenAIErrorRetryable(apiError, undefined, null, undefined)).toBeUndefined();
        });
    });

    describe('Cross-provider compatibility', () => {
        it('should work with Azure OpenAI provider context', () => {
            const openaiError = new RateLimitError(
                429,
                { message: 'Rate limit exceeded' },
                'Rate limit exceeded',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'azure_openai',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error).toBeInstanceOf(LlumiverseError);
            expect(error.code).toBe(429);
            expect(error.retryable).toBe(true);
            expect(error.message).toContain('[azure_openai]');
        });

        it('should work with xAI provider context', () => {
            const openaiError = new BadRequestError(
                400,
                { message: 'Invalid parameter' },
                'Invalid parameter',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'xai',
                model: 'grok-2',
                operation: 'execute',
            });

            expect(error).toBeInstanceOf(LlumiverseError);
            expect(error.code).toBe(400);
            expect(error.retryable).toBe(false);
            expect(error.message).toContain('[xai]');
        });

        it('should work with Azure Foundry provider context', () => {
            const openaiError = new InternalServerError(
                500,
                { message: 'Server error' },
                'Server error',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'azure_foundry',
                model: 'gpt-4',
                operation: 'execute',
            });

            expect(error).toBeInstanceOf(LlumiverseError);
            expect(error.code).toBe(500);
            expect(error.retryable).toBe(true);
            expect(error.message).toContain('[azure_foundry]');
        });

        it('should work with OpenAI-compatible provider context', () => {
            const openaiError = new AuthenticationError(
                401,
                { message: 'Invalid API key' },
                'Invalid API key',
                new Headers()
            );

            const error = driver.formatLlumiverseError( openaiError, {
                provider: 'openai_compatible',
                model: 'custom-model',
                operation: 'execute',
            });

            expect(error).toBeInstanceOf(LlumiverseError);
            expect(error.code).toBe(401);
            expect(error.retryable).toBe(false);
            expect(error.message).toContain('[openai_compatible]');
        });
    });
});