/** * SqlJsBackend - Pure JavaScript SQLite for Windows compatibility * * When better-sqlite3 native compilation fails on Windows, * sql.js provides a WASM-based fallback that works everywhere. * * @module v3/memory/sqljs-backend */ import { EventEmitter } from 'node:events'; import { readFileSync, writeFileSync, existsSync } from 'node:fs'; import initSqlJs from 'sql.js'; /** * Default configuration values */ const DEFAULT_CONFIG = { databasePath: ':memory:', optimize: true, defaultNamespace: 'default', maxEntries: 1000000, verbose: false, autoPersistInterval: 5000, // 5 seconds }; /** * SqlJs Backend for Cross-Platform Memory Storage * * Provides: * - Pure JavaScript/WASM implementation (no native compilation) * - Windows, macOS, Linux compatibility * - Same SQL interface as better-sqlite3 * - In-memory with periodic disk persistence * - Fallback when native SQLite fails */ export class SqlJsBackend extends EventEmitter { config; db = null; initialized = false; persistTimer = null; SQL = null; // Performance tracking stats = { queryCount: 0, totalQueryTime: 0, writeCount: 0, totalWriteTime: 0, }; constructor(config = {}) { super(); this.config = { ...DEFAULT_CONFIG, ...config }; } /** * Initialize the SqlJs backend */ async initialize() { if (this.initialized) return; // Load sql.js WASM this.SQL = await initSqlJs({ locateFile: this.config.wasmPath ? () => this.config.wasmPath : (file) => `https://sql.js.org/dist/${file}`, }); // Load existing database if exists and not in-memory if (this.config.databasePath !== ':memory:' && existsSync(this.config.databasePath)) { const buffer = readFileSync(this.config.databasePath); this.db = new this.SQL.Database(new Uint8Array(buffer)); if (this.config.verbose) { console.log(`[SqlJsBackend] Loaded database from ${this.config.databasePath}`); } } else { // Create new database this.db = new this.SQL.Database(); if (this.config.verbose) { console.log('[SqlJsBackend] Created new in-memory database'); } } // Create schema this.createSchema(); // Set up auto-persist if enabled if (this.config.autoPersistInterval > 0 && this.config.databasePath !== ':memory:') { this.persistTimer = setInterval(() => { this.persist().catch((err) => { this.emit('error', { operation: 'auto-persist', error: err }); }); }, this.config.autoPersistInterval); } this.initialized = true; this.emit('initialized'); } /** * Shutdown the backend */ async shutdown() { if (!this.initialized || !this.db) return; // Stop auto-persist timer if (this.persistTimer) { clearInterval(this.persistTimer); this.persistTimer = null; } // Final persist before shutdown if (this.config.databasePath !== ':memory:') { await this.persist(); } this.db.close(); this.db = null; this.initialized = false; this.emit('shutdown'); } /** * Create database schema */ createSchema() { if (!this.db) return; // Main entries table this.db.run(` CREATE TABLE IF NOT EXISTS memory_entries ( id TEXT PRIMARY KEY, key TEXT NOT NULL, content TEXT NOT NULL, embedding BLOB, type TEXT NOT NULL, namespace TEXT NOT NULL, tags TEXT NOT NULL, metadata TEXT NOT NULL, owner_id TEXT, access_level TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, expires_at INTEGER, version INTEGER NOT NULL DEFAULT 1, "references" TEXT NOT NULL, access_count INTEGER NOT NULL DEFAULT 0, last_accessed_at INTEGER NOT NULL ) `); // Indexes for performance this.db.run('CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace)'); this.db.run('CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key)'); this.db.run('CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type)'); this.db.run('CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at)'); this.db.run('CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at)'); this.db.run('CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key)'); if (this.config.verbose) { console.log('[SqlJsBackend] Schema created successfully'); } } /** * Store a memory entry */ async store(entry) { this.ensureInitialized(); const startTime = performance.now(); const stmt = ` INSERT OR REPLACE INTO memory_entries ( id, key, content, embedding, type, namespace, tags, metadata, owner_id, access_level, created_at, updated_at, expires_at, version, "references", access_count, last_accessed_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `; const embeddingBuffer = entry.embedding ? Buffer.from(entry.embedding.buffer) : null; this.db.run(stmt, [ entry.id, entry.key, entry.content, embeddingBuffer, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId || null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt, ]); const duration = performance.now() - startTime; this.stats.writeCount++; this.stats.totalWriteTime += duration; this.emit('entry:stored', { entry, duration }); } /** * Retrieve a memory entry by ID */ async get(id) { this.ensureInitialized(); const startTime = performance.now(); const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?'); const row = stmt.getAsObject([id]); stmt.free(); const duration = performance.now() - startTime; this.stats.queryCount++; this.stats.totalQueryTime += duration; if (!row || Object.keys(row).length === 0) { return null; } const entry = this.rowToEntry(row); // Update access tracking this.updateAccessTracking(id); this.emit('entry:retrieved', { id, duration }); return entry; } /** * Retrieve a memory entry by key within a namespace */ async getByKey(namespace, key) { this.ensureInitialized(); const startTime = performance.now(); const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE namespace = ? AND key = ?'); const row = stmt.getAsObject([namespace, key]); stmt.free(); const duration = performance.now() - startTime; this.stats.queryCount++; this.stats.totalQueryTime += duration; if (!row || Object.keys(row).length === 0) { return null; } const entry = this.rowToEntry(row); // Update access tracking this.updateAccessTracking(entry.id); this.emit('entry:retrieved', { namespace, key, duration }); return entry; } /** * Update a memory entry */ async update(id, updateData) { this.ensureInitialized(); const startTime = performance.now(); // Get existing entry const existing = await this.get(id); if (!existing) return null; // Merge updates const updated = { ...existing, ...updateData, updatedAt: Date.now(), version: existing.version + 1, }; // Store updated entry await this.store(updated); const duration = performance.now() - startTime; this.emit('entry:updated', { id, update: updateData, duration }); return updated; } /** * Delete a memory entry */ async delete(id) { this.ensureInitialized(); const startTime = performance.now(); this.db.run('DELETE FROM memory_entries WHERE id = ?', [id]); const duration = performance.now() - startTime; this.stats.writeCount++; this.stats.totalWriteTime += duration; this.emit('entry:deleted', { id, duration }); return true; } /** * Query memory entries */ async query(query) { this.ensureInitialized(); const startTime = performance.now(); let sql = 'SELECT * FROM memory_entries WHERE 1=1'; const params = []; // Namespace filter if (query.namespace) { sql += ' AND namespace = ?'; params.push(query.namespace); } // Type filter if (query.memoryType) { sql += ' AND type = ?'; params.push(query.memoryType); } // Owner filter if (query.ownerId) { sql += ' AND owner_id = ?'; params.push(query.ownerId); } // Access level filter if (query.accessLevel) { sql += ' AND access_level = ?'; params.push(query.accessLevel); } // Key filters if (query.key) { sql += ' AND key = ?'; params.push(query.key); } else if (query.keyPrefix) { sql += ' AND key LIKE ?'; params.push(query.keyPrefix + '%'); } // Time range filters if (query.createdAfter) { sql += ' AND created_at >= ?'; params.push(query.createdAfter); } if (query.createdBefore) { sql += ' AND created_at <= ?'; params.push(query.createdBefore); } if (query.updatedAfter) { sql += ' AND updated_at >= ?'; params.push(query.updatedAfter); } if (query.updatedBefore) { sql += ' AND updated_at <= ?'; params.push(query.updatedBefore); } // Expiration filter if (!query.includeExpired) { sql += ' AND (expires_at IS NULL OR expires_at > ?)'; params.push(Date.now()); } // Ordering and pagination sql += ' ORDER BY created_at DESC'; if (query.limit) { sql += ' LIMIT ?'; params.push(query.limit); } if (query.offset) { sql += ' OFFSET ?'; params.push(query.offset); } const stmt = this.db.prepare(sql); if (params.length > 0) { stmt.bind(params); } const results = []; while (stmt.step()) { const row = stmt.getAsObject(); const entry = this.rowToEntry(row); // Tag filtering (post-query since tags are JSON) if (query.tags && query.tags.length > 0) { const hasAllTags = query.tags.every((tag) => entry.tags.includes(tag)); if (!hasAllTags) continue; } // Metadata filtering (post-query since metadata is JSON) if (query.metadata) { const matchesMetadata = Object.entries(query.metadata).every(([key, value]) => entry.metadata[key] === value); if (!matchesMetadata) continue; } results.push(entry); } stmt.free(); const duration = performance.now() - startTime; this.stats.queryCount++; this.stats.totalQueryTime += duration; this.emit('query:executed', { query, resultCount: results.length, duration }); return results; } /** * Semantic vector search (limited without vector index) */ async search(embedding, options) { this.ensureInitialized(); // Get all entries with embeddings const entries = await this.query({ type: 'hybrid', limit: options.filters?.limit || 1000, }); // Calculate cosine similarity for each entry const results = []; for (const entry of entries) { if (!entry.embedding) continue; const similarity = this.cosineSimilarity(embedding, entry.embedding); if (options.threshold && similarity < options.threshold) { continue; } results.push({ entry, score: similarity, distance: 1 - similarity, }); } // Sort by score descending results.sort((a, b) => b.score - a.score); // Return top k results return results.slice(0, options.k); } /** * Bulk insert entries */ async bulkInsert(entries) { this.ensureInitialized(); for (const entry of entries) { await this.store(entry); } this.emit('bulk:inserted', { count: entries.length }); } /** * Bulk delete entries */ async bulkDelete(ids) { this.ensureInitialized(); let count = 0; for (const id of ids) { const success = await this.delete(id); if (success) count++; } this.emit('bulk:deleted', { count }); return count; } /** * Get entry count */ async count(namespace) { this.ensureInitialized(); let sql = 'SELECT COUNT(*) as count FROM memory_entries'; const params = []; if (namespace) { sql += ' WHERE namespace = ?'; params.push(namespace); } const stmt = this.db.prepare(sql); const row = stmt.getAsObject(params); stmt.free(); return row.count || 0; } /** * List all namespaces */ async listNamespaces() { this.ensureInitialized(); const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries'); const namespaces = []; while (stmt.step()) { const row = stmt.getAsObject(); namespaces.push(row.namespace); } stmt.free(); return namespaces; } /** * Clear all entries in a namespace */ async clearNamespace(namespace) { this.ensureInitialized(); const countBefore = await this.count(namespace); this.db.run('DELETE FROM memory_entries WHERE namespace = ?', [namespace]); this.emit('namespace:cleared', { namespace, count: countBefore }); return countBefore; } /** * Get backend statistics */ async getStats() { this.ensureInitialized(); const total = await this.count(); // Count by namespace const entriesByNamespace = {}; const namespaces = await this.listNamespaces(); for (const ns of namespaces) { entriesByNamespace[ns] = await this.count(ns); } // Count by type const entriesByType = {}; const types = ['episodic', 'semantic', 'procedural', 'working', 'cache']; for (const type of types) { const stmt = this.db.prepare('SELECT COUNT(*) as count FROM memory_entries WHERE type = ?'); const row = stmt.getAsObject([type]); stmt.free(); entriesByType[type] = row.count || 0; } return { totalEntries: total, entriesByNamespace, entriesByType, memoryUsage: this.estimateMemoryUsage(), avgQueryTime: this.stats.queryCount > 0 ? this.stats.totalQueryTime / this.stats.queryCount : 0, avgSearchTime: 0, // Not tracked separately }; } /** * Perform health check */ async healthCheck() { const issues = []; const recommendations = []; // Storage health const storageStart = performance.now(); const storageHealthy = this.db !== null; const storageLatency = performance.now() - storageStart; if (!storageHealthy) { issues.push('Database not initialized'); } // Index health (sql.js doesn't have native vector index) const indexHealth = { status: 'healthy', latency: 0, message: 'No vector index (brute-force search)', }; recommendations.push('Consider using better-sqlite3 with HNSW for faster vector search'); // Cache health (not applicable for sql.js) const cacheHealth = { status: 'healthy', latency: 0, message: 'No separate cache layer', }; const status = issues.length === 0 ? 'healthy' : 'degraded'; return { status, components: { storage: { status: storageHealthy ? 'healthy' : 'unhealthy', latency: storageLatency, }, index: indexHealth, cache: cacheHealth, }, timestamp: Date.now(), issues, recommendations, }; } /** * Persist changes to disk (sql.js is in-memory, needs explicit save) */ async persist() { if (!this.db || this.config.databasePath === ':memory:') { return; } const data = this.db.export(); const buffer = Buffer.from(data); writeFileSync(this.config.databasePath, buffer); if (this.config.verbose) { console.log(`[SqlJsBackend] Persisted ${buffer.length} bytes to ${this.config.databasePath}`); } this.emit('persisted', { size: buffer.length, path: this.config.databasePath }); } // ===== Helper Methods ===== ensureInitialized() { if (!this.initialized || !this.db) { throw new Error('SqlJsBackend not initialized. Call initialize() first.'); } } rowToEntry(row) { return { id: row.id, key: row.key, content: row.content, embedding: row.embedding ? new Float32Array(new Uint8Array(row.embedding).buffer) : undefined, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags), metadata: JSON.parse(row.metadata), ownerId: row.owner_id, accessLevel: row.access_level, createdAt: row.created_at, updatedAt: row.updated_at, expiresAt: row.expires_at, version: row.version, references: JSON.parse(row.references), accessCount: row.access_count, lastAccessedAt: row.last_accessed_at, }; } updateAccessTracking(id) { if (!this.db) return; this.db.run('UPDATE memory_entries SET access_count = access_count + 1, last_accessed_at = ? WHERE id = ?', [Date.now(), id]); } cosineSimilarity(a, b) { let dotProduct = 0; let normA = 0; let normB = 0; for (let i = 0; i < a.length; i++) { dotProduct += a[i] * b[i]; normA += a[i] * a[i]; normB += b[i] * b[i]; } if (normA === 0 || normB === 0) return 0; return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); } estimateMemoryUsage() { if (!this.db) return 0; // Export to get size const data = this.db.export(); return data.length; } } //# sourceMappingURL=sqljs-backend.js.map