1484 lines
56 KiB
JavaScript
1484 lines
56 KiB
JavaScript
/**
|
|
* EmbeddingService - Unified embedding interface for agentic-flow
|
|
*
|
|
* Uses ruvector@0.1.61+ for ONNX embeddings with:
|
|
* - SIMD128 acceleration (6x faster)
|
|
* - Parallel worker threads (7 workers)
|
|
* - all-MiniLM-L6-v2 model (384 dimensions)
|
|
* - Persistent SQLite cache (0.1ms vs 400ms)
|
|
*
|
|
* Configure via:
|
|
* - AGENTIC_FLOW_EMBEDDINGS=simple|onnx|auto (default: auto)
|
|
* - AGENTIC_FLOW_EMBEDDING_MODEL=all-MiniLM-L6-v2 (default)
|
|
* - AGENTIC_FLOW_EMBEDDING_CACHE=true|false (default: true)
|
|
* - AGENTIC_FLOW_PERSISTENT_CACHE=true|false (default: true)
|
|
*/
|
|
import { getEmbeddingCache } from './EmbeddingCache.js';
|
|
// ONNX availability cache
|
|
let onnxAvailable = null;
|
|
let ruvectorModule = null;
|
|
/**
|
|
* Detect ONNX/SIMD support by loading ruvector
|
|
*/
|
|
async function detectOnnx() {
|
|
if (onnxAvailable !== null) {
|
|
return onnxAvailable;
|
|
}
|
|
try {
|
|
const mod = await import('ruvector');
|
|
ruvectorModule = mod;
|
|
onnxAvailable = mod.isOnnxAvailable?.() ?? false;
|
|
return onnxAvailable;
|
|
}
|
|
catch (error) {
|
|
// Ruvector loading failed - fall back to simple embeddings
|
|
onnxAvailable = false;
|
|
return false;
|
|
}
|
|
}
|
|
// Simple LRU cache for embeddings (in-memory, fast)
|
|
class LRUCache {
|
|
cache = new Map();
|
|
maxSize;
|
|
constructor(maxSize = 1000) {
|
|
this.maxSize = maxSize;
|
|
}
|
|
get(key) {
|
|
const value = this.cache.get(key);
|
|
if (value) {
|
|
// Move to end (most recently used)
|
|
this.cache.delete(key);
|
|
this.cache.set(key, value);
|
|
}
|
|
return value;
|
|
}
|
|
set(key, value) {
|
|
if (this.cache.size >= this.maxSize) {
|
|
// Delete oldest (first) entry
|
|
const firstKey = this.cache.keys().next().value;
|
|
if (firstKey) {
|
|
this.cache.delete(firstKey);
|
|
}
|
|
}
|
|
this.cache.set(key, value);
|
|
}
|
|
clear() {
|
|
this.cache.clear();
|
|
}
|
|
get size() {
|
|
return this.cache.size;
|
|
}
|
|
}
|
|
export class EmbeddingService {
|
|
static instance = null;
|
|
backend;
|
|
effectiveBackend = null;
|
|
dimension;
|
|
modelName;
|
|
// ONNX state
|
|
modelLoaded = false;
|
|
loadingPromise = null;
|
|
// Stats
|
|
totalEmbeddings = 0;
|
|
totalLatencyMs = 0;
|
|
cacheHits = 0;
|
|
// Cache (in-memory LRU)
|
|
cache;
|
|
cacheEnabled;
|
|
// Persistent cache (SQLite)
|
|
persistentCache = null;
|
|
persistentCacheEnabled;
|
|
// Corpus for search operations
|
|
corpus = { texts: [], embeddings: [] };
|
|
constructor() {
|
|
// Default to 'auto' which will detect ONNX and use it if available
|
|
this.backend = process.env.AGENTIC_FLOW_EMBEDDINGS || 'auto';
|
|
this.modelName = process.env.AGENTIC_FLOW_EMBEDDING_MODEL || 'all-MiniLM-L6-v2';
|
|
this.dimension = 256; // Will be updated when ONNX loads (384)
|
|
this.cacheEnabled = process.env.AGENTIC_FLOW_EMBEDDING_CACHE !== 'false';
|
|
this.persistentCacheEnabled = process.env.AGENTIC_FLOW_PERSISTENT_CACHE !== 'false';
|
|
this.cache = new LRUCache(1000);
|
|
// Initialize persistent cache
|
|
if (this.persistentCacheEnabled) {
|
|
try {
|
|
this.persistentCache = getEmbeddingCache({ dimension: 384 });
|
|
}
|
|
catch (error) {
|
|
console.warn('[EmbeddingService] Persistent cache unavailable:', error);
|
|
this.persistentCacheEnabled = false;
|
|
}
|
|
}
|
|
}
|
|
static getInstance() {
|
|
if (!EmbeddingService.instance) {
|
|
EmbeddingService.instance = new EmbeddingService();
|
|
}
|
|
return EmbeddingService.instance;
|
|
}
|
|
/**
|
|
* Resolve the effective backend based on ONNX detection
|
|
*/
|
|
async resolveBackend() {
|
|
if (this.effectiveBackend) {
|
|
return this.effectiveBackend;
|
|
}
|
|
if (this.backend === 'auto') {
|
|
const hasOnnx = await detectOnnx();
|
|
this.effectiveBackend = hasOnnx ? 'onnx' : 'simple';
|
|
if (hasOnnx) {
|
|
this.dimension = 384; // all-MiniLM-L6-v2 dimension
|
|
}
|
|
}
|
|
else {
|
|
this.effectiveBackend = this.backend;
|
|
if (this.backend === 'onnx') {
|
|
await detectOnnx(); // Ensure module is loaded
|
|
this.dimension = 384;
|
|
}
|
|
}
|
|
return this.effectiveBackend;
|
|
}
|
|
/**
|
|
* Get configured backend (may be 'auto')
|
|
*/
|
|
getBackend() {
|
|
return this.backend;
|
|
}
|
|
/**
|
|
* Get effective backend after detection
|
|
*/
|
|
getEffectiveBackend() {
|
|
return this.effectiveBackend || this.backend;
|
|
}
|
|
/**
|
|
* Get embedding dimension
|
|
*/
|
|
getDimension() {
|
|
return this.dimension;
|
|
}
|
|
/**
|
|
* Check if ONNX model is loaded
|
|
*/
|
|
isModelLoaded() {
|
|
return this.modelLoaded;
|
|
}
|
|
/**
|
|
* Generate embedding for text
|
|
* Auto-detects ONNX and uses it if available (default behavior)
|
|
*/
|
|
async embed(text) {
|
|
const startTime = performance.now();
|
|
// Check in-memory cache first (fastest)
|
|
if (this.cacheEnabled) {
|
|
const cached = this.cache.get(text);
|
|
if (cached) {
|
|
this.cacheHits++;
|
|
return cached;
|
|
}
|
|
}
|
|
// Check persistent cache (SQLite, ~0.1ms)
|
|
if (this.persistentCache) {
|
|
const cached = this.persistentCache.get(text, this.modelName);
|
|
if (cached) {
|
|
this.cacheHits++;
|
|
// Also store in memory cache for faster subsequent access
|
|
if (this.cacheEnabled) {
|
|
this.cache.set(text, cached);
|
|
}
|
|
return cached;
|
|
}
|
|
}
|
|
// Resolve backend (handles 'auto' mode)
|
|
const effectiveBackend = await this.resolveBackend();
|
|
let embedding;
|
|
if (effectiveBackend === 'onnx' && ruvectorModule) {
|
|
const result = await ruvectorModule.embed(text);
|
|
if (result?.embedding) {
|
|
embedding = result.embedding;
|
|
this.modelLoaded = true;
|
|
}
|
|
else {
|
|
embedding = this.simpleEmbed(text);
|
|
}
|
|
}
|
|
else {
|
|
embedding = this.simpleEmbed(text);
|
|
}
|
|
// Update stats
|
|
this.totalEmbeddings++;
|
|
this.totalLatencyMs += performance.now() - startTime;
|
|
// Cache result in memory
|
|
if (this.cacheEnabled) {
|
|
this.cache.set(text, embedding);
|
|
}
|
|
// Cache result persistently (for cross-session)
|
|
if (this.persistentCache && effectiveBackend === 'onnx') {
|
|
this.persistentCache.set(text, embedding, this.modelName);
|
|
}
|
|
return embedding;
|
|
}
|
|
/**
|
|
* Generate embeddings for multiple texts (batch processing with parallel workers)
|
|
* Batch processing provides significant speedup with parallel ONNX workers
|
|
*/
|
|
async embedBatch(texts) {
|
|
const startTime = performance.now();
|
|
// Check cache for all texts first
|
|
if (this.cacheEnabled) {
|
|
const cachedResults = texts.map(t => this.cache.get(t) || null);
|
|
const allCached = cachedResults.every(r => r !== null);
|
|
if (allCached) {
|
|
this.cacheHits += texts.length;
|
|
return cachedResults;
|
|
}
|
|
}
|
|
// Resolve backend
|
|
const effectiveBackend = await this.resolveBackend();
|
|
if (effectiveBackend === 'onnx' && ruvectorModule) {
|
|
const result = await ruvectorModule.embedBatch(texts);
|
|
if (result?.embeddings && result.embeddings.length === texts.length) {
|
|
const embeddings = result.embeddings;
|
|
// Cache individual embeddings
|
|
if (this.cacheEnabled) {
|
|
for (let i = 0; i < texts.length; i++) {
|
|
this.cache.set(texts[i], embeddings[i]);
|
|
}
|
|
}
|
|
// Update stats
|
|
this.totalEmbeddings += texts.length;
|
|
this.totalLatencyMs += performance.now() - startTime;
|
|
this.modelLoaded = true;
|
|
return embeddings;
|
|
}
|
|
}
|
|
// Fall back to sequential for simple backend
|
|
return Promise.all(texts.map(t => this.embed(t)));
|
|
}
|
|
/**
|
|
* Compute similarity between two texts
|
|
*/
|
|
async similarity(text1, text2) {
|
|
const effectiveBackend = await this.resolveBackend();
|
|
if (effectiveBackend === 'onnx' && ruvectorModule) {
|
|
const result = await ruvectorModule.similarity(text1, text2);
|
|
return result.similarity;
|
|
}
|
|
// Fall back to embedding + cosine
|
|
const [e1, e2] = await Promise.all([this.embed(text1), this.embed(text2)]);
|
|
return this.cosineSimilarity(e1, e2);
|
|
}
|
|
/**
|
|
* Compute NxN similarity matrix for a list of texts
|
|
* Uses parallel workers for ONNX backend
|
|
*/
|
|
async similarityMatrix(texts) {
|
|
const embeddings = await this.embedBatch(texts);
|
|
const n = texts.length;
|
|
const matrix = Array(n).fill(null).map(() => Array(n).fill(0));
|
|
for (let i = 0; i < n; i++) {
|
|
matrix[i][i] = 1.0; // Self-similarity
|
|
for (let j = i + 1; j < n; j++) {
|
|
const sim = this.cosineSimilarity(embeddings[i], embeddings[j]);
|
|
matrix[i][j] = sim;
|
|
matrix[j][i] = sim; // Symmetric
|
|
}
|
|
}
|
|
return matrix;
|
|
}
|
|
/**
|
|
* Build a corpus for semantic search
|
|
*/
|
|
async buildCorpus(texts) {
|
|
this.corpus.texts = texts;
|
|
this.corpus.embeddings = await this.embedBatch(texts);
|
|
}
|
|
/**
|
|
* Semantic search against the corpus
|
|
* Returns top-k most similar texts
|
|
*/
|
|
async semanticSearch(query, topK = 5) {
|
|
if (this.corpus.texts.length === 0) {
|
|
throw new Error('Corpus not built. Call buildCorpus() first.');
|
|
}
|
|
const queryEmbedding = await this.embed(query);
|
|
const results = [];
|
|
for (let i = 0; i < this.corpus.texts.length; i++) {
|
|
const sim = this.cosineSimilarity(queryEmbedding, this.corpus.embeddings[i]);
|
|
results.push({
|
|
text: this.corpus.texts[i],
|
|
index: i,
|
|
similarity: sim,
|
|
});
|
|
}
|
|
// Sort by similarity (descending) and return top-k
|
|
results.sort((a, b) => b.similarity - a.similarity);
|
|
return results.slice(0, topK);
|
|
}
|
|
/**
|
|
* Find near-duplicate texts in a list
|
|
* Groups texts with similarity above threshold
|
|
*/
|
|
async findDuplicates(texts, threshold = 0.9) {
|
|
const embeddings = await this.embedBatch(texts);
|
|
const n = texts.length;
|
|
const visited = new Set();
|
|
const groups = [];
|
|
for (let i = 0; i < n; i++) {
|
|
if (visited.has(i))
|
|
continue;
|
|
const group = {
|
|
indices: [i],
|
|
texts: [texts[i]],
|
|
similarity: 1.0,
|
|
};
|
|
for (let j = i + 1; j < n; j++) {
|
|
if (visited.has(j))
|
|
continue;
|
|
const sim = this.cosineSimilarity(embeddings[i], embeddings[j]);
|
|
if (sim >= threshold) {
|
|
group.indices.push(j);
|
|
group.texts.push(texts[j]);
|
|
group.similarity = Math.min(group.similarity, sim);
|
|
visited.add(j);
|
|
}
|
|
}
|
|
if (group.indices.length > 1) {
|
|
visited.add(i);
|
|
groups.push(group);
|
|
}
|
|
}
|
|
return groups;
|
|
}
|
|
/**
|
|
* K-means clustering of texts
|
|
* Returns cluster assignments and centroids
|
|
*/
|
|
async clusterTexts(texts, k = 3, maxIterations = 100) {
|
|
const embeddings = await this.embedBatch(texts);
|
|
const n = texts.length;
|
|
const dim = this.dimension;
|
|
// Initialize centroids randomly (copy to new ArrayBuffer for consistent typing)
|
|
const centroidIndices = new Set();
|
|
while (centroidIndices.size < k && centroidIndices.size < n) {
|
|
centroidIndices.add(Math.floor(Math.random() * n));
|
|
}
|
|
let centroids = Array.from(centroidIndices).map(i => {
|
|
const copy = new Float32Array(dim);
|
|
copy.set(embeddings[i]);
|
|
return copy;
|
|
});
|
|
let clusters = new Array(n).fill(0);
|
|
for (let iter = 0; iter < maxIterations; iter++) {
|
|
// Assign points to nearest centroid
|
|
const newClusters = embeddings.map(emb => {
|
|
let bestCluster = 0;
|
|
let bestSim = -Infinity;
|
|
for (let c = 0; c < k; c++) {
|
|
const sim = this.cosineSimilarity(emb, centroids[c]);
|
|
if (sim > bestSim) {
|
|
bestSim = sim;
|
|
bestCluster = c;
|
|
}
|
|
}
|
|
return bestCluster;
|
|
});
|
|
// Check convergence
|
|
const changed = newClusters.some((c, i) => c !== clusters[i]);
|
|
clusters = newClusters;
|
|
if (!changed)
|
|
break;
|
|
// Update centroids
|
|
const newCentroids = [];
|
|
for (let c = 0; c < k; c++) {
|
|
newCentroids.push(new Float32Array(dim));
|
|
}
|
|
const counts = new Array(k).fill(0);
|
|
for (let i = 0; i < n; i++) {
|
|
const c = clusters[i];
|
|
counts[c]++;
|
|
for (let d = 0; d < dim; d++) {
|
|
newCentroids[c][d] += embeddings[i][d];
|
|
}
|
|
}
|
|
// Normalize centroids
|
|
for (let c = 0; c < k; c++) {
|
|
if (counts[c] > 0) {
|
|
let norm = 0;
|
|
for (let d = 0; d < dim; d++) {
|
|
newCentroids[c][d] /= counts[c];
|
|
norm += newCentroids[c][d] * newCentroids[c][d];
|
|
}
|
|
norm = Math.sqrt(norm) || 1;
|
|
for (let d = 0; d < dim; d++) {
|
|
newCentroids[c][d] /= norm;
|
|
}
|
|
}
|
|
}
|
|
centroids = newCentroids;
|
|
}
|
|
return { clusters, centroids };
|
|
}
|
|
/**
|
|
* Stream embeddings for large batches (memory efficient)
|
|
* Yields embeddings one at a time
|
|
*/
|
|
async *streamEmbed(texts, batchSize = 32) {
|
|
for (let i = 0; i < texts.length; i += batchSize) {
|
|
const batch = texts.slice(i, i + batchSize);
|
|
const embeddings = await this.embedBatch(batch);
|
|
for (let j = 0; j < batch.length; j++) {
|
|
yield {
|
|
index: i + j,
|
|
text: batch[j],
|
|
embedding: embeddings[j],
|
|
};
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* Simple hash-based embedding (fast, not semantic)
|
|
*/
|
|
simpleEmbed(text, dim = 256) {
|
|
const embedding = new Float32Array(dim);
|
|
// Multi-pass hash for better distribution
|
|
for (let i = 0; i < text.length; i++) {
|
|
const code = text.charCodeAt(i);
|
|
embedding[i % dim] += code / 255;
|
|
embedding[(i * 7) % dim] += (code * 0.3) / 255;
|
|
embedding[(i * 13) % dim] += (code * 0.2) / 255;
|
|
}
|
|
// Normalize
|
|
let norm = 0;
|
|
for (let i = 0; i < dim; i++) {
|
|
norm += embedding[i] * embedding[i];
|
|
}
|
|
norm = Math.sqrt(norm) || 1;
|
|
for (let i = 0; i < dim; i++) {
|
|
embedding[i] /= norm;
|
|
}
|
|
return embedding;
|
|
}
|
|
/**
|
|
* Compute cosine similarity between two embeddings
|
|
*/
|
|
cosineSimilarity(a, b) {
|
|
if (ruvectorModule?.cosineSimilarity) {
|
|
return ruvectorModule.cosineSimilarity(a, b);
|
|
}
|
|
// JS fallback
|
|
let dot = 0;
|
|
let normA = 0;
|
|
let normB = 0;
|
|
for (let i = 0; i < a.length; i++) {
|
|
dot += a[i] * b[i];
|
|
normA += a[i] * a[i];
|
|
normB += b[i] * b[i];
|
|
}
|
|
return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
|
|
}
|
|
/**
|
|
* Get statistics
|
|
*/
|
|
getStats() {
|
|
const effective = this.effectiveBackend || this.backend;
|
|
const ruvectorStats = ruvectorModule?.getStats?.() || {};
|
|
// Get persistent cache stats
|
|
let persistentCacheStats;
|
|
if (this.persistentCache) {
|
|
const cacheStats = this.persistentCache.getStats();
|
|
persistentCacheStats = {
|
|
enabled: true,
|
|
entries: cacheStats.totalEntries,
|
|
hits: cacheStats.hits,
|
|
misses: cacheStats.misses,
|
|
hitRate: cacheStats.hitRate,
|
|
dbSizeKB: Math.round(cacheStats.dbSizeBytes / 1024),
|
|
};
|
|
}
|
|
return {
|
|
backend: this.backend,
|
|
effectiveBackend: effective,
|
|
dimension: this.dimension,
|
|
totalEmbeddings: this.totalEmbeddings,
|
|
totalLatencyMs: this.totalLatencyMs,
|
|
avgLatencyMs: this.totalEmbeddings > 0 ? this.totalLatencyMs / this.totalEmbeddings : 0,
|
|
cacheHits: this.cacheHits,
|
|
modelLoaded: this.modelLoaded,
|
|
modelName: effective === 'onnx' ? this.modelName : undefined,
|
|
simdAvailable: ruvectorStats.simdAvailable ?? onnxAvailable,
|
|
parallelWorkers: ruvectorStats.workerCount ?? undefined,
|
|
persistentCache: persistentCacheStats,
|
|
};
|
|
}
|
|
/**
|
|
* Clear in-memory cache
|
|
*/
|
|
clearCache() {
|
|
this.cache.clear();
|
|
}
|
|
/**
|
|
* Clear persistent cache (SQLite)
|
|
*/
|
|
clearPersistentCache() {
|
|
if (this.persistentCache) {
|
|
this.persistentCache.clear();
|
|
}
|
|
}
|
|
/**
|
|
* Clear all caches (memory + persistent)
|
|
*/
|
|
clearAllCaches() {
|
|
this.cache.clear();
|
|
if (this.persistentCache) {
|
|
this.persistentCache.clear();
|
|
}
|
|
}
|
|
/**
|
|
* Get persistent cache stats
|
|
*/
|
|
getPersistentCacheStats() {
|
|
if (!this.persistentCache)
|
|
return null;
|
|
const stats = this.persistentCache.getStats();
|
|
return {
|
|
entries: stats.totalEntries,
|
|
hits: stats.hits,
|
|
misses: stats.misses,
|
|
hitRate: stats.hitRate,
|
|
};
|
|
}
|
|
/**
|
|
* Clear corpus
|
|
*/
|
|
clearCorpus() {
|
|
this.corpus = { texts: [], embeddings: [] };
|
|
}
|
|
/**
|
|
* Shutdown (cleanup workers)
|
|
*/
|
|
async shutdown() {
|
|
if (ruvectorModule?.shutdown) {
|
|
await ruvectorModule.shutdown();
|
|
}
|
|
}
|
|
/**
|
|
* Reset instance (for testing)
|
|
*/
|
|
static async reset() {
|
|
if (EmbeddingService.instance) {
|
|
await EmbeddingService.instance.shutdown();
|
|
}
|
|
EmbeddingService.instance = null;
|
|
onnxAvailable = null;
|
|
ruvectorModule = null;
|
|
}
|
|
/**
|
|
* Pretrain cache with texts from files
|
|
* Embeds content and stores in persistent cache for fast retrieval
|
|
*
|
|
* @param sources - File paths or glob patterns, or array of texts
|
|
* @param options - Pretrain options
|
|
* @returns Stats about pretraining
|
|
*/
|
|
async pretrain(sources, options = {}) {
|
|
const { batchSize = 32, onProgress, chunkSize = 512, overlapSize = 64, skipCached = true } = options;
|
|
const startTime = performance.now();
|
|
let processed = 0;
|
|
let cached = 0;
|
|
let skipped = 0;
|
|
// Resolve texts to embed
|
|
const texts = [];
|
|
if (typeof sources === 'string') {
|
|
sources = [sources];
|
|
}
|
|
for (const source of sources) {
|
|
// Check if it's a file path or glob pattern
|
|
if (source.includes('/') || source.includes('*') || source.includes('.')) {
|
|
try {
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
const { glob } = await import('glob').catch(() => ({ glob: null }));
|
|
// Handle glob patterns
|
|
let files = [];
|
|
if (source.includes('*') && glob) {
|
|
files = await glob(source);
|
|
}
|
|
else if (fs.existsSync(source)) {
|
|
files = [source];
|
|
}
|
|
for (const file of files) {
|
|
try {
|
|
const content = fs.readFileSync(file, 'utf-8');
|
|
// Chunk large files
|
|
if (content.length > chunkSize * 2) {
|
|
for (let i = 0; i < content.length; i += chunkSize - overlapSize) {
|
|
const chunk = content.slice(i, i + chunkSize);
|
|
if (chunk.trim().length > 10) {
|
|
texts.push(chunk);
|
|
}
|
|
}
|
|
}
|
|
else if (content.trim().length > 10) {
|
|
texts.push(content);
|
|
}
|
|
}
|
|
catch {
|
|
// Skip unreadable files
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
// Treat as plain text if file operations fail
|
|
texts.push(source);
|
|
}
|
|
}
|
|
else {
|
|
texts.push(source);
|
|
}
|
|
}
|
|
// Filter out already cached texts
|
|
const toEmbed = [];
|
|
for (const text of texts) {
|
|
if (skipCached && this.persistentCache?.has(text, this.modelName)) {
|
|
skipped++;
|
|
}
|
|
else {
|
|
toEmbed.push(text);
|
|
}
|
|
}
|
|
// Embed in batches
|
|
for (let i = 0; i < toEmbed.length; i += batchSize) {
|
|
const batch = toEmbed.slice(i, i + batchSize);
|
|
const embeddings = await this.embedBatch(batch);
|
|
// Store in persistent cache (embedBatch already handles this for ONNX)
|
|
cached += embeddings.length;
|
|
processed += batch.length;
|
|
if (onProgress) {
|
|
onProgress(processed, toEmbed.length);
|
|
}
|
|
}
|
|
return {
|
|
processed,
|
|
cached,
|
|
skipped,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Pretrain with common programming patterns
|
|
* Pre-caches embeddings for frequently used code patterns
|
|
*/
|
|
async pretrainCodePatterns() {
|
|
const patterns = [
|
|
// Common programming constructs
|
|
'function implementation',
|
|
'class definition',
|
|
'interface declaration',
|
|
'type alias',
|
|
'import statement',
|
|
'export module',
|
|
'async await pattern',
|
|
'promise handling',
|
|
'error handling try catch',
|
|
'conditional logic if else',
|
|
'loop iteration for while',
|
|
'array map filter reduce',
|
|
'object destructuring',
|
|
'spread operator',
|
|
'rest parameters',
|
|
// Code operations
|
|
'refactor code',
|
|
'fix bug',
|
|
'add feature',
|
|
'write tests',
|
|
'add documentation',
|
|
'optimize performance',
|
|
'improve readability',
|
|
'handle edge cases',
|
|
'add validation',
|
|
'implement authentication',
|
|
// File types
|
|
'TypeScript file',
|
|
'JavaScript module',
|
|
'React component',
|
|
'Vue component',
|
|
'CSS stylesheet',
|
|
'JSON configuration',
|
|
'Markdown documentation',
|
|
'Python script',
|
|
'Shell script',
|
|
'SQL query',
|
|
// Agent routing patterns
|
|
'code review task',
|
|
'architecture design',
|
|
'testing strategy',
|
|
'debugging session',
|
|
'performance analysis',
|
|
'security audit',
|
|
'documentation update',
|
|
'API design',
|
|
'database schema',
|
|
'deployment configuration',
|
|
];
|
|
const startTime = performance.now();
|
|
const embeddings = await this.embedBatch(patterns);
|
|
return {
|
|
cached: embeddings.length,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Pretrain from repository structure
|
|
* Analyzes file names and paths to pre-cache common patterns
|
|
*/
|
|
async pretrainFromRepo(repoPath = '.') {
|
|
const startTime = performance.now();
|
|
let files = 0;
|
|
let chunks = 0;
|
|
try {
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
// Common code file extensions
|
|
const extensions = ['.ts', '.tsx', '.js', '.jsx', '.py', '.md', '.json'];
|
|
const walkDir = (dir) => {
|
|
try {
|
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
// Skip node_modules, .git, etc.
|
|
if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
|
|
walkDir(fullPath);
|
|
}
|
|
}
|
|
else if (extensions.some(ext => entry.name.endsWith(ext))) {
|
|
return fullPath;
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
// Skip unreadable directories
|
|
}
|
|
return null;
|
|
};
|
|
// Collect files
|
|
const filePaths = [];
|
|
const collectFiles = (dir) => {
|
|
try {
|
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
|
|
collectFiles(fullPath);
|
|
}
|
|
}
|
|
else if (extensions.some(ext => entry.name.endsWith(ext))) {
|
|
filePaths.push(fullPath);
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
// Skip unreadable
|
|
}
|
|
};
|
|
collectFiles(repoPath);
|
|
files = filePaths.length;
|
|
// Pretrain from collected files
|
|
if (filePaths.length > 0) {
|
|
const result = await this.pretrain(filePaths, {
|
|
batchSize: 16,
|
|
chunkSize: 512,
|
|
overlapSize: 64,
|
|
});
|
|
chunks = result.cached;
|
|
}
|
|
}
|
|
catch (err) {
|
|
// Repository analysis failed
|
|
}
|
|
return {
|
|
files,
|
|
chunks,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Incremental pretrain - only process changed files since last run
|
|
* Uses git diff to detect modified files
|
|
*/
|
|
async pretrainIncremental(options = {}) {
|
|
const { since = 'HEAD~10', repoPath = '.' } = options;
|
|
const startTime = performance.now();
|
|
let changedFiles = 0;
|
|
let newChunks = 0;
|
|
let skipped = 0;
|
|
try {
|
|
const { execSync } = await import('child_process');
|
|
const path = await import('path');
|
|
const fs = await import('fs');
|
|
// Get changed files from git
|
|
const gitOutput = execSync(`git diff --name-only ${since}`, {
|
|
cwd: repoPath,
|
|
encoding: 'utf-8',
|
|
});
|
|
const changedPaths = gitOutput
|
|
.split('\n')
|
|
.filter(f => f.trim())
|
|
.map(f => path.join(repoPath, f))
|
|
.filter(f => {
|
|
try {
|
|
return fs.existsSync(f) && fs.statSync(f).isFile();
|
|
}
|
|
catch {
|
|
return false;
|
|
}
|
|
});
|
|
changedFiles = changedPaths.length;
|
|
if (changedPaths.length > 0) {
|
|
const result = await this.pretrain(changedPaths, {
|
|
batchSize: 16,
|
|
chunkSize: 512,
|
|
overlapSize: 64,
|
|
skipCached: true,
|
|
});
|
|
newChunks = result.cached;
|
|
skipped = result.skipped;
|
|
}
|
|
}
|
|
catch {
|
|
// Git not available or not a repo
|
|
}
|
|
return {
|
|
changedFiles,
|
|
newChunks,
|
|
skipped,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Smart chunking - split code by semantic boundaries
|
|
* (functions, classes, etc.) instead of fixed size
|
|
*/
|
|
semanticChunk(content, fileType) {
|
|
const chunks = [];
|
|
// TypeScript/JavaScript patterns
|
|
if (['.ts', '.tsx', '.js', '.jsx'].some(ext => fileType.endsWith(ext))) {
|
|
// Split on function/class/interface boundaries
|
|
const patterns = [
|
|
/^(export\s+)?(async\s+)?function\s+\w+/gm,
|
|
/^(export\s+)?class\s+\w+/gm,
|
|
/^(export\s+)?interface\s+\w+/gm,
|
|
/^(export\s+)?type\s+\w+/gm,
|
|
/^(export\s+)?const\s+\w+\s*=/gm,
|
|
];
|
|
let lastIndex = 0;
|
|
const boundaries = [0];
|
|
for (const pattern of patterns) {
|
|
let match;
|
|
while ((match = pattern.exec(content)) !== null) {
|
|
boundaries.push(match.index);
|
|
}
|
|
}
|
|
boundaries.push(content.length);
|
|
boundaries.sort((a, b) => a - b);
|
|
// Extract chunks between boundaries
|
|
for (let i = 0; i < boundaries.length - 1; i++) {
|
|
const chunk = content.slice(boundaries[i], boundaries[i + 1]).trim();
|
|
if (chunk.length > 20 && chunk.length < 2000) {
|
|
chunks.push(chunk);
|
|
}
|
|
}
|
|
}
|
|
// Python patterns
|
|
else if (fileType.endsWith('.py')) {
|
|
const patterns = [
|
|
/^(async\s+)?def\s+\w+/gm,
|
|
/^class\s+\w+/gm,
|
|
];
|
|
const boundaries = [0];
|
|
for (const pattern of patterns) {
|
|
let match;
|
|
while ((match = pattern.exec(content)) !== null) {
|
|
boundaries.push(match.index);
|
|
}
|
|
}
|
|
boundaries.push(content.length);
|
|
boundaries.sort((a, b) => a - b);
|
|
for (let i = 0; i < boundaries.length - 1; i++) {
|
|
const chunk = content.slice(boundaries[i], boundaries[i + 1]).trim();
|
|
if (chunk.length > 20 && chunk.length < 2000) {
|
|
chunks.push(chunk);
|
|
}
|
|
}
|
|
}
|
|
// Markdown - split by headers
|
|
else if (fileType.endsWith('.md')) {
|
|
const sections = content.split(/^#+\s+/gm);
|
|
for (const section of sections) {
|
|
if (section.trim().length > 20) {
|
|
chunks.push(section.trim().slice(0, 1000));
|
|
}
|
|
}
|
|
}
|
|
// Fallback to fixed-size chunking
|
|
if (chunks.length === 0) {
|
|
const chunkSize = 512;
|
|
const overlap = 64;
|
|
for (let i = 0; i < content.length; i += chunkSize - overlap) {
|
|
const chunk = content.slice(i, i + chunkSize);
|
|
if (chunk.trim().length > 20) {
|
|
chunks.push(chunk);
|
|
}
|
|
}
|
|
}
|
|
return chunks;
|
|
}
|
|
/**
|
|
* Pretrain with semantic chunking
|
|
* Uses code structure to create meaningful chunks
|
|
*/
|
|
async pretrainSemantic(sources, options = {}) {
|
|
const { batchSize = 32, onProgress } = options;
|
|
const startTime = performance.now();
|
|
let fileCount = 0;
|
|
let chunkCount = 0;
|
|
const allChunks = [];
|
|
try {
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
for (const source of sources) {
|
|
if (fs.existsSync(source)) {
|
|
try {
|
|
const content = fs.readFileSync(source, 'utf-8');
|
|
const ext = path.extname(source);
|
|
const chunks = this.semanticChunk(content, ext);
|
|
allChunks.push(...chunks);
|
|
fileCount++;
|
|
}
|
|
catch {
|
|
// Skip unreadable files
|
|
}
|
|
}
|
|
}
|
|
// Embed and cache all chunks
|
|
for (let i = 0; i < allChunks.length; i += batchSize) {
|
|
const batch = allChunks.slice(i, i + batchSize);
|
|
await this.embedBatch(batch);
|
|
chunkCount += batch.length;
|
|
if (onProgress) {
|
|
onProgress(chunkCount, allChunks.length);
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
// Pretrain failed
|
|
}
|
|
return {
|
|
files: fileCount,
|
|
chunks: chunkCount,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Priority pretrain - cache most frequently used patterns first
|
|
* Tracks access patterns and prioritizes high-frequency queries
|
|
*/
|
|
accessCounts = new Map();
|
|
recordAccess(text) {
|
|
this.accessCounts.set(text, (this.accessCounts.get(text) || 0) + 1);
|
|
}
|
|
getTopPatterns(n = 100) {
|
|
return Array.from(this.accessCounts.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, n)
|
|
.map(([text]) => text);
|
|
}
|
|
async pretrainPriority(n = 100) {
|
|
const topPatterns = this.getTopPatterns(n);
|
|
const startTime = performance.now();
|
|
if (topPatterns.length > 0) {
|
|
await this.embedBatch(topPatterns);
|
|
}
|
|
return {
|
|
cached: topPatterns.length,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Warmup cache on session start
|
|
* Combines code patterns + recent repo changes
|
|
*/
|
|
async warmup(repoPath = '.') {
|
|
const startTime = performance.now();
|
|
// First: load common patterns
|
|
const patternResult = await this.pretrainCodePatterns();
|
|
// Second: load recent git changes
|
|
const incrementalResult = await this.pretrainIncremental({
|
|
since: 'HEAD~5',
|
|
repoPath,
|
|
});
|
|
return {
|
|
patterns: patternResult.cached,
|
|
recentChanges: incrementalResult.newChunks,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Intelligent pretrain using ruvector worker pool
|
|
* Analyzes repo structure, code patterns, and prepares cache
|
|
* Uses parallel workers for maximum throughput
|
|
*/
|
|
async pretrainIntelligent(options = {}) {
|
|
const { repoPath = '.', parallel = true, onProgress } = options;
|
|
const startTime = performance.now();
|
|
const stages = {
|
|
codePatterns: { count: 0, timeMs: 0 },
|
|
astAnalysis: { files: 0, functions: 0, timeMs: 0 },
|
|
gitHistory: { commits: 0, hotFiles: 0, timeMs: 0 },
|
|
dependencies: { modules: 0, imports: 0, timeMs: 0 },
|
|
semanticChunks: { chunks: 0, timeMs: 0 },
|
|
};
|
|
let totalCached = 0;
|
|
try {
|
|
// Stage 1: Code patterns (common programming patterns)
|
|
onProgress?.('codePatterns', 0);
|
|
const stage1Start = performance.now();
|
|
const patternResult = await this.pretrainCodePatterns();
|
|
stages.codePatterns = {
|
|
count: patternResult.cached,
|
|
timeMs: performance.now() - stage1Start,
|
|
};
|
|
totalCached += patternResult.cached;
|
|
onProgress?.('codePatterns', 100);
|
|
// Stage 2: AST Analysis using ruvector workers (if available)
|
|
onProgress?.('astAnalysis', 0);
|
|
const stage2Start = performance.now();
|
|
try {
|
|
if (ruvectorModule && parallel) {
|
|
// Use ruvector's analyzeFilesParallel if available
|
|
const mod = ruvectorModule;
|
|
if (mod.analyzeFilesParallel) {
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
// Collect source files
|
|
const sourceFiles = [];
|
|
const collectSources = (dir) => {
|
|
try {
|
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
|
|
collectSources(fullPath);
|
|
}
|
|
}
|
|
else if (['.ts', '.tsx', '.js', '.jsx'].some(ext => entry.name.endsWith(ext))) {
|
|
sourceFiles.push(fullPath);
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
};
|
|
collectSources(repoPath);
|
|
// Analyze in parallel
|
|
const astResult = await mod.analyzeFilesParallel(sourceFiles.slice(0, 100));
|
|
stages.astAnalysis = {
|
|
files: sourceFiles.length,
|
|
functions: astResult?.functions || 0,
|
|
timeMs: performance.now() - stage2Start,
|
|
};
|
|
// Extract function signatures for caching
|
|
if (astResult?.signatures) {
|
|
await this.embedBatch(astResult.signatures.slice(0, 200));
|
|
totalCached += Math.min(astResult.signatures.length, 200);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
onProgress?.('astAnalysis', 100);
|
|
// Stage 3: Git history analysis (hot files = frequently changed)
|
|
onProgress?.('gitHistory', 0);
|
|
const stage3Start = performance.now();
|
|
try {
|
|
const { execSync } = await import('child_process');
|
|
// Get commit count
|
|
const commitCount = execSync('git rev-list --count HEAD', {
|
|
cwd: repoPath,
|
|
encoding: 'utf-8',
|
|
}).trim();
|
|
// Get hot files (most frequently changed)
|
|
const hotFilesOutput = execSync('git log --format="" --name-only -n 100 | sort | uniq -c | sort -rn | head -20', { cwd: repoPath, encoding: 'utf-8' });
|
|
const hotFiles = hotFilesOutput
|
|
.split('\n')
|
|
.filter(l => l.trim())
|
|
.map(l => l.trim().split(/\s+/).slice(1).join(' '))
|
|
.filter(f => f);
|
|
stages.gitHistory = {
|
|
commits: parseInt(commitCount) || 0,
|
|
hotFiles: hotFiles.length,
|
|
timeMs: performance.now() - stage3Start,
|
|
};
|
|
// Pretrain hot files
|
|
if (hotFiles.length > 0) {
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
const validFiles = hotFiles
|
|
.map(f => path.join(repoPath, f))
|
|
.filter(f => fs.existsSync(f));
|
|
if (validFiles.length > 0) {
|
|
const result = await this.pretrainSemantic(validFiles, { batchSize: 16 });
|
|
totalCached += result.chunks;
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
onProgress?.('gitHistory', 100);
|
|
// Stage 4: Dependency analysis
|
|
onProgress?.('dependencies', 0);
|
|
const stage4Start = performance.now();
|
|
try {
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
// Parse package.json for dependencies
|
|
const pkgPath = path.join(repoPath, 'package.json');
|
|
if (fs.existsSync(pkgPath)) {
|
|
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
|
|
const deps = Object.keys(pkg.dependencies || {});
|
|
const devDeps = Object.keys(pkg.devDependencies || {});
|
|
const allDeps = [...deps, ...devDeps];
|
|
stages.dependencies = {
|
|
modules: allDeps.length,
|
|
imports: 0,
|
|
timeMs: performance.now() - stage4Start,
|
|
};
|
|
// Cache dependency names for import resolution
|
|
if (allDeps.length > 0) {
|
|
const depPatterns = allDeps.map(d => `import from ${d}`);
|
|
await this.embedBatch(depPatterns);
|
|
totalCached += depPatterns.length;
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
onProgress?.('dependencies', 100);
|
|
// Stage 5: Semantic chunking with parallel embedding
|
|
onProgress?.('semanticChunks', 0);
|
|
const stage5Start = performance.now();
|
|
try {
|
|
const incrementalResult = await this.pretrainIncremental({
|
|
since: 'HEAD~20',
|
|
repoPath,
|
|
});
|
|
stages.semanticChunks = {
|
|
chunks: incrementalResult.newChunks,
|
|
timeMs: performance.now() - stage5Start,
|
|
};
|
|
totalCached += incrementalResult.newChunks;
|
|
}
|
|
catch { }
|
|
onProgress?.('semanticChunks', 100);
|
|
}
|
|
catch (err) {
|
|
// Pretrain failed, return partial results
|
|
}
|
|
return {
|
|
stages,
|
|
totalCached,
|
|
totalTimeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Background pretrain - runs in worker if available
|
|
* Non-blocking, returns immediately with a promise
|
|
*/
|
|
pretrainBackground(options = {}) {
|
|
let cancelled = false;
|
|
const promise = (async () => {
|
|
if (cancelled)
|
|
return;
|
|
// Run warmup in background
|
|
await this.warmup(options.repoPath);
|
|
if (cancelled)
|
|
return;
|
|
// Then run intelligent pretrain
|
|
await this.pretrainIntelligent({
|
|
...options,
|
|
parallel: true,
|
|
});
|
|
})();
|
|
return {
|
|
promise,
|
|
cancel: () => { cancelled = true; },
|
|
};
|
|
}
|
|
/**
|
|
* AI-enhanced pretrain using ruvector attention mechanisms
|
|
* Uses HyperbolicAttention for code structure, MoE for routing
|
|
*/
|
|
async pretrainWithAI(options = {}) {
|
|
const { repoPath = '.', attentionType = 'auto', onProgress } = options;
|
|
const startTime = performance.now();
|
|
const patterns = [];
|
|
let totalCached = 0;
|
|
let attentionInfo = { type: 'none', timeMs: 0 };
|
|
let predictions = { prefetch: 0, confidence: 0 };
|
|
try {
|
|
const mod = ruvectorModule;
|
|
// Step 1: Determine best attention type for codebase
|
|
onProgress?.('attention', 'Selecting optimal attention mechanism...');
|
|
let selectedAttention = attentionType;
|
|
if (attentionType === 'auto' && mod) {
|
|
// Use getAttentionForUseCase if available
|
|
if (mod.getAttentionForUseCase) {
|
|
const result = await mod.getAttentionForUseCase('code_analysis');
|
|
selectedAttention = result?.type || 'hyperbolic';
|
|
}
|
|
else {
|
|
// Default to hyperbolic for hierarchical code structure
|
|
selectedAttention = 'hyperbolic';
|
|
}
|
|
}
|
|
attentionInfo.type = selectedAttention;
|
|
const attentionStart = performance.now();
|
|
// Step 2: Use attention to identify important code regions
|
|
onProgress?.('analysis', `Using ${selectedAttention} attention for code analysis...`);
|
|
if (mod) {
|
|
// Collect code samples for attention-based analysis
|
|
const fs = await import('fs');
|
|
const path = await import('path');
|
|
const codeSamples = [];
|
|
const collectCode = (dir, maxFiles = 50) => {
|
|
if (codeSamples.length >= maxFiles)
|
|
return;
|
|
try {
|
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
if (codeSamples.length >= maxFiles)
|
|
break;
|
|
const fullPath = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
if (!entry.name.startsWith('.') && entry.name !== 'node_modules' && entry.name !== 'dist') {
|
|
collectCode(fullPath, maxFiles);
|
|
}
|
|
}
|
|
else if (['.ts', '.tsx', '.js', '.jsx'].some(ext => entry.name.endsWith(ext))) {
|
|
try {
|
|
const content = fs.readFileSync(fullPath, 'utf-8');
|
|
if (content.length < 5000) {
|
|
codeSamples.push(content);
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
};
|
|
collectCode(repoPath);
|
|
// Step 3: Use attention mechanisms to weight code importance
|
|
if (mod.HyperbolicAttention && selectedAttention === 'hyperbolic') {
|
|
try {
|
|
// Hyperbolic attention for hierarchical code structure
|
|
const attention = new mod.HyperbolicAttention({ dim: 384 });
|
|
// Identify structural patterns (classes, functions, imports)
|
|
const structuralPatterns = [
|
|
'class definition with constructor',
|
|
'async function with error handling',
|
|
'interface with multiple properties',
|
|
'type with generics',
|
|
'import statement block',
|
|
'export default component',
|
|
'hook implementation useEffect',
|
|
'API endpoint handler',
|
|
'database query function',
|
|
'authentication middleware',
|
|
];
|
|
await this.embedBatch(structuralPatterns);
|
|
patterns.push({ type: 'structural', count: structuralPatterns.length });
|
|
totalCached += structuralPatterns.length;
|
|
}
|
|
catch { }
|
|
}
|
|
if (mod.MoEAttention && selectedAttention === 'moe') {
|
|
try {
|
|
// MoE for routing different code patterns to experts
|
|
const routingPatterns = [
|
|
// Expert 1: Frontend
|
|
'React component with state',
|
|
'Vue component with props',
|
|
'CSS styling module',
|
|
// Expert 2: Backend
|
|
'Express route handler',
|
|
'GraphQL resolver',
|
|
'REST API endpoint',
|
|
// Expert 3: Data
|
|
'SQL query builder',
|
|
'MongoDB aggregation',
|
|
'Redis cache operation',
|
|
// Expert 4: Testing
|
|
'Jest test case',
|
|
'E2E test scenario',
|
|
'Mock implementation',
|
|
];
|
|
await this.embedBatch(routingPatterns);
|
|
patterns.push({ type: 'routing', count: routingPatterns.length });
|
|
totalCached += routingPatterns.length;
|
|
}
|
|
catch { }
|
|
}
|
|
if (mod.GraphRoPeAttention && selectedAttention === 'graph') {
|
|
try {
|
|
// Graph attention for dependency understanding
|
|
const graphPatterns = [
|
|
'module exports',
|
|
'circular dependency',
|
|
'shared utility import',
|
|
'type re-export',
|
|
'barrel file index',
|
|
'lazy import dynamic',
|
|
'peer dependency',
|
|
'optional dependency',
|
|
];
|
|
await this.embedBatch(graphPatterns);
|
|
patterns.push({ type: 'graph', count: graphPatterns.length });
|
|
totalCached += graphPatterns.length;
|
|
}
|
|
catch { }
|
|
}
|
|
attentionInfo.timeMs = performance.now() - attentionStart;
|
|
// Step 4: FastGRNN for pattern prediction (if available)
|
|
onProgress?.('prediction', 'Training pattern predictor...');
|
|
if (mod.FastGRNN) {
|
|
try {
|
|
// Use recent access patterns to predict what's needed next
|
|
const topPatterns = this.getTopPatterns(50);
|
|
if (topPatterns.length > 0) {
|
|
// Prefetch predicted patterns
|
|
const prefetchPatterns = [
|
|
...topPatterns.slice(0, 20),
|
|
// Add related patterns
|
|
...topPatterns.slice(0, 10).map(p => `similar to: ${p}`),
|
|
];
|
|
await this.embedBatch(prefetchPatterns);
|
|
predictions = {
|
|
prefetch: prefetchPatterns.length,
|
|
confidence: 0.85, // Estimated based on access history
|
|
};
|
|
totalCached += prefetchPatterns.length;
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
}
|
|
// Step 5: Standard warmup
|
|
onProgress?.('warmup', 'Running standard warmup...');
|
|
const warmupResult = await this.warmup(repoPath);
|
|
totalCached += warmupResult.patterns + warmupResult.recentChanges;
|
|
patterns.push({ type: 'warmup', count: warmupResult.patterns + warmupResult.recentChanges });
|
|
}
|
|
catch (err) {
|
|
// AI pretrain failed, continue with basic
|
|
}
|
|
return {
|
|
patterns,
|
|
attention: attentionInfo,
|
|
predictions,
|
|
totalCached,
|
|
totalTimeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
/**
|
|
* Context-aware prefetch using attention
|
|
* Predicts what embeddings will be needed based on current context
|
|
*/
|
|
async prefetchForContext(context) {
|
|
const startTime = performance.now();
|
|
let prefetched = 0;
|
|
let confidence = 0;
|
|
try {
|
|
const patterns = [];
|
|
// Add patterns based on current file type
|
|
if (context.currentFile) {
|
|
const ext = context.currentFile.split('.').pop() || '';
|
|
const filePatterns = {
|
|
ts: ['TypeScript type checking', 'interface implementation', 'generic types'],
|
|
tsx: ['React component', 'JSX rendering', 'hook usage'],
|
|
js: ['JavaScript module', 'CommonJS require', 'ES6 import'],
|
|
jsx: ['React component', 'JSX element', 'props handling'],
|
|
py: ['Python function', 'class method', 'import statement'],
|
|
md: ['documentation', 'README section', 'code example'],
|
|
};
|
|
patterns.push(...(filePatterns[ext] || []));
|
|
}
|
|
// Add patterns based on task type
|
|
if (context.taskType) {
|
|
const taskPatterns = {
|
|
edit: ['code modification', 'variable rename', 'function update'],
|
|
review: ['code review', 'bug detection', 'style check'],
|
|
debug: ['error trace', 'stack analysis', 'variable inspection'],
|
|
test: ['test case', 'assertion', 'mock setup'],
|
|
refactor: ['code cleanup', 'pattern extraction', 'abstraction'],
|
|
};
|
|
patterns.push(...(taskPatterns[context.taskType] || []));
|
|
}
|
|
// Add patterns based on user query similarity
|
|
if (context.userQuery) {
|
|
patterns.push(context.userQuery);
|
|
// Add variations
|
|
patterns.push(`how to ${context.userQuery}`);
|
|
patterns.push(`implement ${context.userQuery}`);
|
|
}
|
|
if (patterns.length > 0) {
|
|
await this.embedBatch(patterns);
|
|
prefetched = patterns.length;
|
|
confidence = Math.min(0.9, 0.5 + patterns.length * 0.05);
|
|
}
|
|
}
|
|
catch {
|
|
// Prefetch failed
|
|
}
|
|
return {
|
|
prefetched,
|
|
confidence,
|
|
timeMs: performance.now() - startTime,
|
|
};
|
|
}
|
|
}
|
|
// Export singleton getter
|
|
export function getEmbeddingService() {
|
|
return EmbeddingService.getInstance();
|
|
}
|
|
// Export convenience functions
|
|
export async function embed(text) {
|
|
return getEmbeddingService().embed(text);
|
|
}
|
|
export async function embedBatch(texts) {
|
|
return getEmbeddingService().embedBatch(texts);
|
|
}
|
|
export async function pretrainCodePatterns() {
|
|
return getEmbeddingService().pretrainCodePatterns();
|
|
}
|
|
export async function pretrainFromRepo(repoPath = '.') {
|
|
return getEmbeddingService().pretrainFromRepo(repoPath);
|
|
}
|
|
export async function textSimilarity(text1, text2) {
|
|
return getEmbeddingService().similarity(text1, text2);
|
|
}
|
|
export function simpleEmbed(text, dim = 256) {
|
|
return getEmbeddingService().simpleEmbed(text, dim);
|
|
}
|
|
export async function similarityMatrix(texts) {
|
|
return getEmbeddingService().similarityMatrix(texts);
|
|
}
|
|
export async function semanticSearch(query, topK = 5) {
|
|
return getEmbeddingService().semanticSearch(query, topK);
|
|
}
|
|
export async function findDuplicates(texts, threshold = 0.9) {
|
|
return getEmbeddingService().findDuplicates(texts, threshold);
|
|
}
|
|
export async function clusterTexts(texts, k = 3) {
|
|
return getEmbeddingService().clusterTexts(texts, k);
|
|
}
|
|
//# sourceMappingURL=EmbeddingService.js.map
|