435 lines
16 KiB
JavaScript
435 lines
16 KiB
JavaScript
/**
|
|
* Provider Manager - Intelligent LLM Provider Fallback & Dynamic Switching
|
|
*
|
|
* Features:
|
|
* - Automatic failover between providers
|
|
* - Health monitoring and circuit breaking
|
|
* - Cost-based optimization
|
|
* - Performance tracking
|
|
* - Rate limit handling
|
|
* - Provider quality scoring
|
|
*/
|
|
import { logger } from '../utils/logger.js';
|
|
export class ProviderManager {
|
|
providers = new Map();
|
|
health = new Map();
|
|
metrics = new Map();
|
|
strategy;
|
|
currentProvider = null;
|
|
healthCheckIntervals = new Map();
|
|
constructor(providers, strategy = {
|
|
type: 'priority',
|
|
maxFailures: 3,
|
|
recoveryTime: 60000, // 1 minute
|
|
retryBackoff: 'exponential'
|
|
}) {
|
|
this.strategy = strategy;
|
|
this.initializeProviders(providers);
|
|
this.startHealthChecks();
|
|
}
|
|
/**
|
|
* Initialize provider configurations
|
|
*/
|
|
initializeProviders(providers) {
|
|
for (const provider of providers) {
|
|
this.providers.set(provider.name, provider);
|
|
// Initialize health tracking
|
|
this.health.set(provider.name, {
|
|
provider: provider.name,
|
|
isHealthy: true,
|
|
lastChecked: new Date(),
|
|
consecutiveFailures: 0,
|
|
averageLatency: 0,
|
|
successRate: 1.0,
|
|
errorRate: 0.0,
|
|
circuitBreakerOpen: false
|
|
});
|
|
// Initialize metrics
|
|
this.metrics.set(provider.name, {
|
|
provider: provider.name,
|
|
totalRequests: 0,
|
|
successfulRequests: 0,
|
|
failedRequests: 0,
|
|
totalTokens: 0,
|
|
totalCost: 0,
|
|
averageLatency: 0,
|
|
lastUsed: new Date()
|
|
});
|
|
}
|
|
logger.info('Provider Manager initialized', {
|
|
providers: Array.from(this.providers.keys()),
|
|
strategy: this.strategy.type
|
|
});
|
|
}
|
|
/**
|
|
* Start health check monitoring for all providers
|
|
*/
|
|
startHealthChecks() {
|
|
for (const [name, config] of this.providers.entries()) {
|
|
if (config.healthCheckInterval) {
|
|
const interval = setInterval(() => {
|
|
this.performHealthCheck(name);
|
|
}, config.healthCheckInterval);
|
|
this.healthCheckIntervals.set(name, interval);
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* Perform health check on a provider
|
|
*/
|
|
async performHealthCheck(provider) {
|
|
const health = this.health.get(provider);
|
|
if (!health)
|
|
return;
|
|
try {
|
|
const startTime = Date.now();
|
|
// TODO: Implement actual health check API call
|
|
// For now, just check if circuit breaker should recover
|
|
const latency = Date.now() - startTime;
|
|
health.lastChecked = new Date();
|
|
health.averageLatency = (health.averageLatency + latency) / 2;
|
|
// Recovery logic: reset circuit breaker after recovery time
|
|
if (health.circuitBreakerOpen) {
|
|
const timeSinceLastCheck = Date.now() - health.lastChecked.getTime();
|
|
if (timeSinceLastCheck >= this.strategy.recoveryTime) {
|
|
logger.info('Circuit breaker recovering', { provider });
|
|
health.circuitBreakerOpen = false;
|
|
health.consecutiveFailures = 0;
|
|
}
|
|
}
|
|
logger.debug('Health check completed', { provider, health });
|
|
}
|
|
catch (error) {
|
|
logger.error('Health check failed', { provider, error: error.message });
|
|
this.recordFailure(provider, error);
|
|
}
|
|
}
|
|
/**
|
|
* Select the best provider based on strategy
|
|
*/
|
|
async selectProvider(taskComplexity, estimatedTokens) {
|
|
const availableProviders = this.getAvailableProviders();
|
|
if (availableProviders.length === 0) {
|
|
throw new Error('No healthy providers available');
|
|
}
|
|
let selectedProvider;
|
|
switch (this.strategy.type) {
|
|
case 'priority':
|
|
selectedProvider = this.selectByPriority(availableProviders);
|
|
break;
|
|
case 'cost-optimized':
|
|
selectedProvider = this.selectByCost(availableProviders, estimatedTokens);
|
|
break;
|
|
case 'performance-optimized':
|
|
selectedProvider = this.selectByPerformance(availableProviders);
|
|
break;
|
|
case 'round-robin':
|
|
selectedProvider = this.selectRoundRobin(availableProviders);
|
|
break;
|
|
default:
|
|
selectedProvider = availableProviders[0];
|
|
}
|
|
// Apply task complexity heuristics
|
|
if (taskComplexity === 'complex' && this.providers.has('anthropic')) {
|
|
const anthropicHealth = this.health.get('anthropic');
|
|
if (anthropicHealth?.isHealthy && !anthropicHealth.circuitBreakerOpen) {
|
|
selectedProvider = 'anthropic'; // Prefer Claude for complex tasks
|
|
}
|
|
}
|
|
else if (taskComplexity === 'simple' && this.providers.has('gemini')) {
|
|
const geminiHealth = this.health.get('gemini');
|
|
if (geminiHealth?.isHealthy && !geminiHealth.circuitBreakerOpen) {
|
|
selectedProvider = 'gemini'; // Prefer Gemini for simple tasks (faster, cheaper)
|
|
}
|
|
}
|
|
this.currentProvider = selectedProvider;
|
|
logger.info('Provider selected', {
|
|
provider: selectedProvider,
|
|
strategy: this.strategy.type,
|
|
taskComplexity,
|
|
estimatedTokens
|
|
});
|
|
return selectedProvider;
|
|
}
|
|
/**
|
|
* Get list of healthy, available providers
|
|
*/
|
|
getAvailableProviders() {
|
|
const available = [];
|
|
for (const [name, config] of this.providers.entries()) {
|
|
const health = this.health.get(name);
|
|
if (config.enabled &&
|
|
health?.isHealthy &&
|
|
!health.circuitBreakerOpen) {
|
|
available.push(name);
|
|
}
|
|
}
|
|
// Sort by priority
|
|
available.sort((a, b) => {
|
|
const priorityA = this.providers.get(a)?.priority ?? 999;
|
|
const priorityB = this.providers.get(b)?.priority ?? 999;
|
|
return priorityA - priorityB;
|
|
});
|
|
return available;
|
|
}
|
|
/**
|
|
* Select provider by priority
|
|
*/
|
|
selectByPriority(providers) {
|
|
return providers[0]; // Already sorted by priority
|
|
}
|
|
/**
|
|
* Select provider by cost optimization
|
|
*/
|
|
selectByCost(providers, estimatedTokens) {
|
|
if (!estimatedTokens) {
|
|
return this.selectByPriority(providers);
|
|
}
|
|
let bestProvider = providers[0];
|
|
let lowestCost = Infinity;
|
|
for (const provider of providers) {
|
|
const config = this.providers.get(provider);
|
|
if (!config)
|
|
continue;
|
|
const estimatedCost = (estimatedTokens / 1_000_000) * config.costPerToken;
|
|
if (estimatedCost < lowestCost) {
|
|
lowestCost = estimatedCost;
|
|
bestProvider = provider;
|
|
}
|
|
}
|
|
return bestProvider;
|
|
}
|
|
/**
|
|
* Select provider by performance (latency + success rate)
|
|
*/
|
|
selectByPerformance(providers) {
|
|
let bestProvider = providers[0];
|
|
let bestScore = -Infinity;
|
|
for (const provider of providers) {
|
|
const health = this.health.get(provider);
|
|
if (!health)
|
|
continue;
|
|
// Score = success rate (weighted 70%) - normalized latency (weighted 30%)
|
|
const normalizedLatency = health.averageLatency / 1000; // Convert to seconds
|
|
const score = (health.successRate * 0.7) - (normalizedLatency * 0.3);
|
|
if (score > bestScore) {
|
|
bestScore = score;
|
|
bestProvider = provider;
|
|
}
|
|
}
|
|
return bestProvider;
|
|
}
|
|
/**
|
|
* Select provider using round-robin
|
|
*/
|
|
selectRoundRobin(providers) {
|
|
if (!this.currentProvider || !providers.includes(this.currentProvider)) {
|
|
return providers[0];
|
|
}
|
|
const currentIndex = providers.indexOf(this.currentProvider);
|
|
const nextIndex = (currentIndex + 1) % providers.length;
|
|
return providers[nextIndex];
|
|
}
|
|
/**
|
|
* Execute request with automatic fallback
|
|
*/
|
|
async executeWithFallback(requestFn, taskComplexity, estimatedTokens) {
|
|
const startTime = Date.now();
|
|
let attempts = 0;
|
|
let lastError = null;
|
|
// Try each available provider
|
|
const availableProviders = this.getAvailableProviders();
|
|
const provider = await this.selectProvider(taskComplexity, estimatedTokens);
|
|
// Start with selected provider, then try fallbacks
|
|
const providersToTry = [
|
|
provider,
|
|
...availableProviders.filter(p => p !== provider)
|
|
];
|
|
for (const currentProvider of providersToTry) {
|
|
attempts++;
|
|
const config = this.providers.get(currentProvider);
|
|
if (!config)
|
|
continue;
|
|
try {
|
|
logger.info('Attempting request', { provider: currentProvider, attempt: attempts });
|
|
const requestStartTime = Date.now();
|
|
const result = await this.executeWithRetry(requestFn, currentProvider, config.maxRetries);
|
|
const latency = Date.now() - requestStartTime;
|
|
// Record success
|
|
this.recordSuccess(currentProvider, latency, estimatedTokens || 0);
|
|
logger.info('Request successful', {
|
|
provider: currentProvider,
|
|
attempts,
|
|
latency,
|
|
totalTime: Date.now() - startTime
|
|
});
|
|
return { result, provider: currentProvider, attempts };
|
|
}
|
|
catch (error) {
|
|
lastError = error;
|
|
logger.warn('Provider request failed', {
|
|
provider: currentProvider,
|
|
attempt: attempts,
|
|
error: lastError.message
|
|
});
|
|
// Record failure
|
|
this.recordFailure(currentProvider, lastError);
|
|
// Continue to next provider
|
|
continue;
|
|
}
|
|
}
|
|
// All providers failed
|
|
throw new Error(`All providers failed after ${attempts} attempts. Last error: ${lastError?.message}`);
|
|
}
|
|
/**
|
|
* Execute request with retry logic
|
|
*/
|
|
async executeWithRetry(requestFn, provider, maxRetries) {
|
|
let lastError = null;
|
|
for (let retry = 0; retry <= maxRetries; retry++) {
|
|
try {
|
|
// Apply backoff delay
|
|
if (retry > 0) {
|
|
const delay = this.calculateBackoffDelay(retry);
|
|
logger.debug('Retry backoff', { provider, retry, delay });
|
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
}
|
|
return await requestFn(provider);
|
|
}
|
|
catch (error) {
|
|
lastError = error;
|
|
// Check if error is retryable
|
|
if (!this.isRetryableError(error)) {
|
|
throw error;
|
|
}
|
|
logger.debug('Retryable error, continuing', {
|
|
provider,
|
|
retry,
|
|
error: lastError.message
|
|
});
|
|
}
|
|
}
|
|
throw lastError || new Error('Max retries exceeded');
|
|
}
|
|
/**
|
|
* Calculate backoff delay
|
|
*/
|
|
calculateBackoffDelay(retry) {
|
|
if (this.strategy.retryBackoff === 'exponential') {
|
|
// Exponential backoff: 1s, 2s, 4s, 8s, 16s...
|
|
return Math.min(1000 * Math.pow(2, retry), 30000); // Max 30s
|
|
}
|
|
else {
|
|
// Linear backoff: 1s, 2s, 3s, 4s...
|
|
return Math.min(1000 * retry, 10000); // Max 10s
|
|
}
|
|
}
|
|
/**
|
|
* Check if error is retryable
|
|
*/
|
|
isRetryableError(error) {
|
|
const retryablePatterns = [
|
|
/rate limit/i,
|
|
/timeout/i,
|
|
/connection/i,
|
|
/network/i,
|
|
/503/,
|
|
/502/,
|
|
/429/
|
|
];
|
|
return retryablePatterns.some(pattern => pattern.test(error.message));
|
|
}
|
|
/**
|
|
* Record successful request
|
|
*/
|
|
recordSuccess(provider, latency, tokens) {
|
|
const metrics = this.metrics.get(provider);
|
|
const health = this.health.get(provider);
|
|
const config = this.providers.get(provider);
|
|
if (metrics) {
|
|
metrics.totalRequests++;
|
|
metrics.successfulRequests++;
|
|
metrics.totalTokens += tokens;
|
|
metrics.totalCost += (tokens / 1_000_000) * (config?.costPerToken || 0);
|
|
metrics.averageLatency = (metrics.averageLatency + latency) / 2;
|
|
metrics.lastUsed = new Date();
|
|
}
|
|
if (health) {
|
|
health.consecutiveFailures = 0;
|
|
health.isHealthy = true;
|
|
health.circuitBreakerOpen = false;
|
|
health.successRate = metrics ? metrics.successfulRequests / metrics.totalRequests : 1.0;
|
|
health.errorRate = 1.0 - health.successRate;
|
|
health.averageLatency = latency;
|
|
}
|
|
}
|
|
/**
|
|
* Record failed request
|
|
*/
|
|
recordFailure(provider, error) {
|
|
const metrics = this.metrics.get(provider);
|
|
const health = this.health.get(provider);
|
|
if (metrics) {
|
|
metrics.totalRequests++;
|
|
metrics.failedRequests++;
|
|
}
|
|
if (health) {
|
|
health.consecutiveFailures++;
|
|
health.successRate = metrics ? metrics.successfulRequests / metrics.totalRequests : 0.0;
|
|
health.errorRate = 1.0 - health.successRate;
|
|
// Open circuit breaker if threshold exceeded
|
|
if (health.consecutiveFailures >= this.strategy.maxFailures) {
|
|
logger.warn('Circuit breaker opened', {
|
|
provider,
|
|
consecutiveFailures: health.consecutiveFailures,
|
|
threshold: this.strategy.maxFailures
|
|
});
|
|
health.circuitBreakerOpen = true;
|
|
health.isHealthy = false;
|
|
}
|
|
}
|
|
logger.error('Provider failure recorded', {
|
|
provider,
|
|
error: error.message,
|
|
consecutiveFailures: health?.consecutiveFailures,
|
|
circuitBreakerOpen: health?.circuitBreakerOpen
|
|
});
|
|
}
|
|
/**
|
|
* Get current metrics for all providers
|
|
*/
|
|
getMetrics() {
|
|
return Array.from(this.metrics.values());
|
|
}
|
|
/**
|
|
* Get health status for all providers
|
|
*/
|
|
getHealth() {
|
|
return Array.from(this.health.values());
|
|
}
|
|
/**
|
|
* Get cost summary
|
|
*/
|
|
getCostSummary() {
|
|
let total = 0;
|
|
let totalTokens = 0;
|
|
const byProvider = {};
|
|
for (const [name, metrics] of this.metrics.entries()) {
|
|
total += metrics.totalCost;
|
|
totalTokens += metrics.totalTokens;
|
|
byProvider[name] = metrics.totalCost;
|
|
}
|
|
return { total, byProvider: byProvider, totalTokens };
|
|
}
|
|
/**
|
|
* Cleanup and stop health checks
|
|
*/
|
|
destroy() {
|
|
for (const interval of this.healthCheckIntervals.values()) {
|
|
clearInterval(interval);
|
|
}
|
|
this.healthCheckIntervals.clear();
|
|
logger.info('Provider Manager destroyed');
|
|
}
|
|
}
|
|
//# sourceMappingURL=provider-manager.js.map
|