/** * Provider Fallback Validation Test * * Tests: * - ProviderManager initialization * - Provider selection strategies * - Automatic fallback * - Circuit breaker * - Cost tracking * - Health monitoring */ import { ProviderManager, ProviderConfig } from '../src/core/provider-manager.js'; import { LongRunningAgent } from '../src/core/long-running-agent.js'; // Test configuration const TEST_PROVIDERS: ProviderConfig[] = [ { name: 'gemini', apiKey: process.env.GOOGLE_GEMINI_API_KEY || 'test-key', priority: 1, maxRetries: 2, timeout: 5000, costPerToken: 0.00015, enabled: true }, { name: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY || 'test-key', priority: 2, maxRetries: 2, timeout: 5000, costPerToken: 0.003, enabled: true }, { name: 'onnx', priority: 3, maxRetries: 1, timeout: 10000, costPerToken: 0, enabled: true } ]; async function testProviderManager() { console.log('๐Ÿงช Test 1: ProviderManager Initialization'); console.log('==========================================\n'); const manager = new ProviderManager(TEST_PROVIDERS, { type: 'priority', maxFailures: 2, recoveryTime: 5000, retryBackoff: 'exponential' }); // Test provider selection const provider = await manager.selectProvider('simple', 100); console.log(`โœ… Selected provider: ${provider}\n`); // Test health status const health = manager.getHealth(); console.log('๐Ÿ“Š Provider Health:'); health.forEach(h => { console.log(` ${h.provider}: healthy=${h.isHealthy}, circuitBreaker=${h.circuitBreakerOpen ? 'OPEN' : 'CLOSED'}`); }); console.log(''); manager.destroy(); console.log('โœ… Test 1 Passed\n'); } async function testFallbackStrategy() { console.log('๐Ÿงช Test 2: Fallback Strategy'); console.log('=============================\n'); const manager = new ProviderManager(TEST_PROVIDERS, { type: 'cost-optimized', maxFailures: 2, recoveryTime: 5000, retryBackoff: 'exponential' }); // Test cost-optimized selection console.log('Testing cost-optimized selection...'); const cheapProvider = await manager.selectProvider('simple', 10000); console.log(`โœ… Cost-optimized provider: ${cheapProvider} (should prefer Gemini/ONNX)\n`); // Test complex task selection const complexProvider = await manager.selectProvider('complex', 5000); console.log(`โœ… Complex task provider: ${complexProvider} (should prefer Anthropic if available)\n`); manager.destroy(); console.log('โœ… Test 2 Passed\n'); } async function testCircuitBreaker() { console.log('๐Ÿงช Test 3: Circuit Breaker'); console.log('===========================\n'); const manager = new ProviderManager( [ { name: 'gemini', priority: 1, maxRetries: 1, timeout: 1000, costPerToken: 0.00015, enabled: true }, { name: 'onnx', priority: 2, maxRetries: 1, timeout: 1000, costPerToken: 0, enabled: true } ], { type: 'priority', maxFailures: 2, // Open circuit after 2 failures recoveryTime: 5000, retryBackoff: 'exponential' } ); let attemptCount = 0; // Simulate failures to trigger circuit breaker try { await manager.executeWithFallback(async (provider) => { attemptCount++; console.log(` Attempt ${attemptCount} with provider: ${provider}`); if (provider === 'gemini' && attemptCount <= 3) { throw new Error('Simulated rate limit error'); } return { success: true, provider }; }); console.log('โœ… Fallback successful after circuit breaker\n'); } catch (error) { console.log(`โš ๏ธ Expected error after all providers failed: ${(error as Error).message}\n`); } // Check circuit breaker status const health = manager.getHealth(); const geminiHealth = health.find(h => h.provider === 'gemini'); if (geminiHealth) { console.log('Circuit Breaker Status:'); console.log(` Gemini circuit breaker: ${geminiHealth.circuitBreakerOpen ? 'OPEN โœ…' : 'CLOSED'}`); console.log(` Consecutive failures: ${geminiHealth.consecutiveFailures}`); console.log(''); } manager.destroy(); console.log('โœ… Test 3 Passed\n'); } async function testCostTracking() { console.log('๐Ÿงช Test 4: Cost Tracking'); console.log('=========================\n'); const manager = new ProviderManager(TEST_PROVIDERS, { type: 'cost-optimized', maxFailures: 3, recoveryTime: 5000, retryBackoff: 'exponential' }); // Execute multiple requests for (let i = 0; i < 3; i++) { await manager.executeWithFallback(async (provider) => { console.log(` Request ${i + 1} using ${provider}`); return { provider, tokens: 1000 }; }, 'simple', 1000); } // Check cost summary const costs = manager.getCostSummary(); console.log('\n๐Ÿ’ฐ Cost Summary:'); console.log(` Total Cost: $${costs.total.toFixed(6)}`); console.log(` Total Tokens: ${costs.totalTokens.toLocaleString()}`); console.log(' By Provider:'); for (const [provider, cost] of Object.entries(costs.byProvider)) { console.log(` ${provider}: $${cost.toFixed(6)}`); } console.log(''); manager.destroy(); console.log('โœ… Test 4 Passed\n'); } async function testLongRunningAgent() { console.log('๐Ÿงช Test 5: Long-Running Agent'); console.log('==============================\n'); const agent = new LongRunningAgent({ agentName: 'test-agent', providers: TEST_PROVIDERS, fallbackStrategy: { type: 'cost-optimized', maxFailures: 2, recoveryTime: 5000, retryBackoff: 'exponential' }, checkpointInterval: 10000, maxRuntime: 60000, costBudget: 1.00 }); await agent.start(); // Execute test tasks try { const task1 = await agent.executeTask({ name: 'test-task-1', complexity: 'simple', estimatedTokens: 500, execute: async (provider) => { console.log(` Task 1 using ${provider}`); return { result: 'success', provider }; } }); console.log(`โœ… Task 1 completed with ${task1.provider}\n`); const task2 = await agent.executeTask({ name: 'test-task-2', complexity: 'medium', estimatedTokens: 1500, execute: async (provider) => { console.log(` Task 2 using ${provider}`); return { result: 'success', provider }; } }); console.log(`โœ… Task 2 completed with ${task2.provider}\n`); } catch (error) { console.error('โŒ Task execution error:', (error as Error).message); } // Get status const status = agent.getStatus(); console.log('๐Ÿ“Š Agent Status:'); console.log(` Running: ${status.isRunning}`); console.log(` Runtime: ${status.runtime}ms`); console.log(` Completed Tasks: ${status.completedTasks}`); console.log(` Failed Tasks: ${status.failedTasks}`); console.log(` Total Cost: $${status.totalCost.toFixed(6)}`); console.log(''); await agent.stop(); console.log('โœ… Test 5 Passed\n'); } async function main() { console.log('\n๐Ÿš€ Provider Fallback Validation Suite'); console.log('======================================\n'); try { await testProviderManager(); await testFallbackStrategy(); await testCircuitBreaker(); await testCostTracking(); await testLongRunningAgent(); console.log('โœ… All tests passed!\n'); process.exit(0); } catch (error) { console.error('\nโŒ Test suite failed:', error); process.exit(1); } } // Run tests if (import.meta.url === `file://${process.argv[1]}`) { main().catch(console.error); } export { main as runProviderFallbackTests };