/** * SQLite Memory Backend * * Provides structured storage for memory entries using SQLite. * Optimized for ACID transactions, exact matches, and complex queries. * Part of ADR-009: Hybrid Memory Backend (SQLite + AgentDB) * * @module v3/memory/sqlite-backend */ import { EventEmitter } from 'node:events'; import Database from 'better-sqlite3'; /** * Default configuration values */ const DEFAULT_CONFIG = { databasePath: ':memory:', walMode: true, optimize: true, defaultNamespace: 'default', maxEntries: 1000000, verbose: false, }; /** * SQLite Backend for Structured Memory Storage * * Provides: * - ACID transactions for data consistency * - Efficient indexing for exact matches and prefix queries * - Full-text search capabilities * - Complex SQL queries with joins and aggregations * - Persistent storage with WAL mode */ export class SQLiteBackend extends EventEmitter { config; db = null; initialized = false; // Performance tracking stats = { queryCount: 0, totalQueryTime: 0, writeCount: 0, totalWriteTime: 0, }; constructor(config = {}) { super(); this.config = { ...DEFAULT_CONFIG, ...config }; } /** * Initialize the SQLite backend */ async initialize() { if (this.initialized) return; // Open database connection this.db = new Database(this.config.databasePath, { verbose: this.config.verbose ? console.log : undefined, }); // Enable WAL mode for better concurrency if (this.config.walMode) { this.db.pragma('journal_mode = WAL'); } // Performance optimizations if (this.config.optimize) { this.db.pragma('synchronous = NORMAL'); this.db.pragma('cache_size = 10000'); this.db.pragma('temp_store = MEMORY'); } // Create schema this.createSchema(); this.initialized = true; this.emit('initialized'); } /** * Shutdown the backend */ async shutdown() { if (!this.initialized || !this.db) return; // Optimize database before closing if (this.config.optimize) { this.db.pragma('optimize'); } this.db.close(); this.db = null; this.initialized = false; this.emit('shutdown'); } /** * Store a memory entry */ async store(entry) { this.ensureInitialized(); const startTime = performance.now(); const stmt = this.db.prepare(` INSERT OR REPLACE INTO memory_entries ( id, key, content, type, namespace, tags, metadata, owner_id, access_level, created_at, updated_at, expires_at, version, "references", access_count, last_accessed_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); stmt.run(entry.id, entry.key, entry.content, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId || null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt); // Store embedding separately (as BLOB) if (entry.embedding) { const embeddingStmt = this.db.prepare(` INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?) `); embeddingStmt.run(entry.id, Buffer.from(entry.embedding.buffer)); } const duration = performance.now() - startTime; this.stats.writeCount++; this.stats.totalWriteTime += duration; this.emit('entry:stored', { id: entry.id, duration }); } /** * Get a memory entry by ID */ async get(id) { this.ensureInitialized(); const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?'); const row = stmt.get(id); if (!row) return null; return this.rowToEntry(row); } /** * Get a memory entry by key within a namespace */ async getByKey(namespace, key) { this.ensureInitialized(); const stmt = this.db.prepare(` SELECT * FROM memory_entries WHERE namespace = ? AND key = ? `); const row = stmt.get(namespace, key); if (!row) return null; return this.rowToEntry(row); } /** * Update a memory entry */ async update(id, update) { this.ensureInitialized(); const entry = await this.get(id); if (!entry) return null; // Apply updates if (update.content !== undefined) entry.content = update.content; if (update.tags !== undefined) entry.tags = update.tags; if (update.metadata !== undefined) { entry.metadata = { ...entry.metadata, ...update.metadata }; } if (update.accessLevel !== undefined) entry.accessLevel = update.accessLevel; if (update.expiresAt !== undefined) entry.expiresAt = update.expiresAt; if (update.references !== undefined) entry.references = update.references; entry.updatedAt = Date.now(); entry.version++; // Store updated entry await this.store(entry); this.emit('entry:updated', { id }); return entry; } /** * Delete a memory entry */ async delete(id) { this.ensureInitialized(); const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?'); const deleteEmbedding = this.db.prepare('DELETE FROM memory_embeddings WHERE entry_id = ?'); const result = deleteEntry.run(id); deleteEmbedding.run(id); if (result.changes > 0) { this.emit('entry:deleted', { id }); return true; } return false; } /** * Query memory entries with filters */ async query(query) { this.ensureInitialized(); const startTime = performance.now(); let sql = 'SELECT * FROM memory_entries WHERE 1=1'; const params = []; // Build WHERE clauses if (query.namespace) { sql += ' AND namespace = ?'; params.push(query.namespace); } if (query.key) { sql += ' AND key = ?'; params.push(query.key); } if (query.keyPrefix) { sql += ' AND key LIKE ?'; params.push(`${query.keyPrefix}%`); } if (query.memoryType) { sql += ' AND type = ?'; params.push(query.memoryType); } if (query.accessLevel) { sql += ' AND access_level = ?'; params.push(query.accessLevel); } if (query.ownerId) { sql += ' AND owner_id = ?'; params.push(query.ownerId); } if (query.createdAfter) { sql += ' AND created_at >= ?'; params.push(query.createdAfter); } if (query.createdBefore) { sql += ' AND created_at <= ?'; params.push(query.createdBefore); } if (query.updatedAfter) { sql += ' AND updated_at >= ?'; params.push(query.updatedAfter); } if (query.updatedBefore) { sql += ' AND updated_at <= ?'; params.push(query.updatedBefore); } if (!query.includeExpired) { sql += ' AND (expires_at IS NULL OR expires_at > ?)'; params.push(Date.now()); } // Tag filtering (safe parameterized query) if (query.tags && query.tags.length > 0) { // Validate tags before using in query for (const tag of query.tags) { if (typeof tag !== 'string' || !/^[a-zA-Z0-9_\-.:]+$/.test(tag)) { throw new Error(`Invalid tag format: ${tag}`); } } // Use parameterized query with JSON functions const tagPlaceholders = query.tags.map(() => '?').join(', '); sql += ` AND EXISTS ( SELECT 1 FROM json_each(tags) AS t WHERE t.value IN (${tagPlaceholders}) )`; params.push(...query.tags); } // Pagination sql += ' ORDER BY created_at DESC'; if (query.limit) { sql += ' LIMIT ?'; params.push(query.limit); } if (query.offset) { sql += ' OFFSET ?'; params.push(query.offset); } const stmt = this.db.prepare(sql); const rows = stmt.all(...params); const results = rows.map((row) => this.rowToEntry(row)); const duration = performance.now() - startTime; this.stats.queryCount++; this.stats.totalQueryTime += duration; return results; } /** * Semantic vector search (not optimized for SQLite, returns empty) * Use HybridBackend for semantic search with AgentDB */ async search(embedding, options) { // SQLite is not optimized for vector search // This method returns empty to encourage use of HybridBackend console.warn('SQLiteBackend.search(): Vector search not optimized. Use HybridBackend for semantic search.'); return []; } /** * Bulk insert entries */ async bulkInsert(entries) { this.ensureInitialized(); const transaction = this.db.transaction((entries) => { for (const entry of entries) { this.storeSync(entry); } }); transaction(entries); this.emit('bulk:inserted', { count: entries.length }); } /** * Bulk delete entries */ async bulkDelete(ids) { this.ensureInitialized(); const deleteEntry = this.db.prepare('DELETE FROM memory_entries WHERE id = ?'); const deleteEmbedding = this.db.prepare('DELETE FROM memory_embeddings WHERE entry_id = ?'); const transaction = this.db.transaction((ids) => { let deleted = 0; for (const id of ids) { const result = deleteEntry.run(id); deleteEmbedding.run(id); if (result.changes > 0) deleted++; } return deleted; }); return transaction(ids); } /** * Get entry count */ async count(namespace) { this.ensureInitialized(); let sql = 'SELECT COUNT(*) as count FROM memory_entries'; const params = []; if (namespace) { sql += ' WHERE namespace = ?'; params.push(namespace); } const stmt = this.db.prepare(sql); const result = stmt.get(...params); return result.count; } /** * List all namespaces */ async listNamespaces() { this.ensureInitialized(); const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries'); const rows = stmt.all(); return rows.map((row) => row.namespace); } /** * Clear all entries in a namespace */ async clearNamespace(namespace) { this.ensureInitialized(); const deleteEntries = this.db.prepare('DELETE FROM memory_entries WHERE namespace = ?'); const result = deleteEntries.run(namespace); // Clean up orphaned embeddings this.db.prepare(` DELETE FROM memory_embeddings WHERE entry_id NOT IN (SELECT id FROM memory_entries) `).run(); return result.changes; } /** * Get backend statistics */ async getStats() { this.ensureInitialized(); // Count by namespace const namespaceStmt = this.db.prepare(` SELECT namespace, COUNT(*) as count FROM memory_entries GROUP BY namespace `); const namespaceRows = namespaceStmt.all(); const entriesByNamespace = {}; for (const row of namespaceRows) { entriesByNamespace[row.namespace] = row.count; } // Count by type const typeStmt = this.db.prepare(` SELECT type, COUNT(*) as count FROM memory_entries GROUP BY type `); const typeRows = typeStmt.all(); const entriesByType = { episodic: 0, semantic: 0, procedural: 0, working: 0, cache: 0, }; for (const row of typeRows) { entriesByType[row.type] = row.count; } // Get database size const pageCount = this.db.pragma('page_count', { simple: true }); const pageSize = this.db.pragma('page_size', { simple: true }); const memoryUsage = pageCount * pageSize; const totalEntries = await this.count(); return { totalEntries, entriesByNamespace, entriesByType, memoryUsage, avgQueryTime: this.stats.queryCount > 0 ? this.stats.totalQueryTime / this.stats.queryCount : 0, avgSearchTime: 0, // Not applicable for SQLite }; } /** * Perform health check */ async healthCheck() { const issues = []; const recommendations = []; if (!this.initialized || !this.db) { return { status: 'unhealthy', components: { storage: { status: 'unhealthy', latency: 0, message: 'Not initialized' }, index: { status: 'healthy', latency: 0 }, cache: { status: 'healthy', latency: 0 }, }, timestamp: Date.now(), issues: ['Backend not initialized'], recommendations: ['Call initialize() before using'], }; } // Check database integrity let storageHealth; try { const integrityCheck = this.db.pragma('integrity_check', { simple: true }); if (integrityCheck === 'ok') { storageHealth = { status: 'healthy', latency: 0 }; } else { issues.push('Database integrity check failed'); recommendations.push('Run VACUUM to repair database'); storageHealth = { status: 'unhealthy', latency: 0, message: 'Integrity check failed' }; } } catch (error) { issues.push('Failed to check database integrity'); storageHealth = { status: 'unhealthy', latency: 0, message: String(error) }; } // Check utilization const totalEntries = await this.count(); const utilizationPercent = (totalEntries / this.config.maxEntries) * 100; if (utilizationPercent > 95) { issues.push('Storage utilization critical (>95%)'); recommendations.push('Cleanup old data or increase maxEntries'); storageHealth = { status: 'unhealthy', latency: 0, message: 'Near capacity' }; } else if (utilizationPercent > 80) { issues.push('Storage utilization high (>80%)'); recommendations.push('Consider cleanup'); if (storageHealth.status === 'healthy') { storageHealth = { status: 'degraded', latency: 0, message: 'High utilization' }; } } const status = storageHealth.status === 'unhealthy' ? 'unhealthy' : storageHealth.status === 'degraded' ? 'degraded' : 'healthy'; return { status, components: { storage: storageHealth, index: { status: 'healthy', latency: 0 }, cache: { status: 'healthy', latency: 0 }, }, timestamp: Date.now(), issues, recommendations, }; } // ===== Private Methods ===== ensureInitialized() { if (!this.initialized || !this.db) { throw new Error('SQLiteBackend not initialized. Call initialize() first.'); } } createSchema() { if (!this.db) return; // Main entries table this.db.exec(` CREATE TABLE IF NOT EXISTS memory_entries ( id TEXT PRIMARY KEY, key TEXT NOT NULL, content TEXT NOT NULL, type TEXT NOT NULL, namespace TEXT NOT NULL, tags TEXT NOT NULL, metadata TEXT NOT NULL, owner_id TEXT, access_level TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, expires_at INTEGER, version INTEGER NOT NULL, "references" TEXT NOT NULL, access_count INTEGER NOT NULL, last_accessed_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace); CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key); CREATE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key); CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type); CREATE INDEX IF NOT EXISTS idx_owner_id ON memory_entries(owner_id); CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at); CREATE INDEX IF NOT EXISTS idx_updated_at ON memory_entries(updated_at); CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at); CREATE TABLE IF NOT EXISTS memory_embeddings ( entry_id TEXT PRIMARY KEY, embedding BLOB, FOREIGN KEY (entry_id) REFERENCES memory_entries(id) ON DELETE CASCADE ); `); } rowToEntry(row) { // Get embedding if exists let embedding; const embeddingStmt = this.db.prepare('SELECT embedding FROM memory_embeddings WHERE entry_id = ?'); const embeddingRow = embeddingStmt.get(row.id); if (embeddingRow && embeddingRow.embedding) { embedding = new Float32Array(embeddingRow.embedding.buffer); } return { id: row.id, key: row.key, content: row.content, embedding, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags), metadata: JSON.parse(row.metadata), ownerId: row.owner_id, accessLevel: row.access_level, createdAt: row.created_at, updatedAt: row.updated_at, expiresAt: row.expires_at, version: row.version, references: JSON.parse(row.references), accessCount: row.access_count, lastAccessedAt: row.last_accessed_at, }; } /** * Synchronous store for use in transactions */ storeSync(entry) { const stmt = this.db.prepare(` INSERT OR REPLACE INTO memory_entries ( id, key, content, type, namespace, tags, metadata, owner_id, access_level, created_at, updated_at, expires_at, version, "references", access_count, last_accessed_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); stmt.run(entry.id, entry.key, entry.content, entry.type, entry.namespace, JSON.stringify(entry.tags), JSON.stringify(entry.metadata), entry.ownerId || null, entry.accessLevel, entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt); if (entry.embedding) { const embeddingStmt = this.db.prepare(` INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?) `); embeddingStmt.run(entry.id, Buffer.from(entry.embedding.buffer)); } } } export default SQLiteBackend; //# sourceMappingURL=sqlite-backend.js.map