/** * RVF Migration Utility — bidirectional migration between RVF and legacy * formats (JSON files, sql.js / better-sqlite3 databases). * @module @claude-flow/memory/rvf-migration */ import { readFile, writeFile, rename, mkdir } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { RvfBackend } from './rvf-backend.js'; import { generateMemoryId } from './types.js'; // -- Internal helpers ------------------------------------------------------- function fillDefaults(raw) { const now = Date.now(); return { id: raw.id ?? generateMemoryId(), key: raw.key ?? '', content: raw.content ?? '', type: raw.type ?? 'semantic', namespace: raw.namespace ?? 'default', tags: raw.tags ?? [], metadata: raw.metadata ?? {}, ownerId: raw.ownerId, accessLevel: raw.accessLevel ?? 'private', createdAt: raw.createdAt ?? now, updatedAt: raw.updatedAt ?? now, expiresAt: raw.expiresAt, version: raw.version ?? 1, references: raw.references ?? [], accessCount: raw.accessCount ?? 0, lastAccessedAt: raw.lastAccessedAt ?? now, embedding: deserializeEmbedding(raw.embedding), }; } function deserializeEmbedding(value) { if (!value) return undefined; if (value instanceof Float32Array) return value; if (value instanceof Buffer || value instanceof Uint8Array) { if (value.byteLength === 0) return undefined; const out = new Float32Array(value.byteLength / 4); const view = new DataView(value.buffer, value.byteOffset, value.byteLength); for (let i = 0; i < out.length; i++) out[i] = view.getFloat32(i * 4, true); return out; } if (Array.isArray(value)) return new Float32Array(value); return undefined; } function serializeForJson(entry) { return { ...entry, embedding: entry.embedding ? Array.from(entry.embedding) : undefined }; } function validateMigrationPath(p) { if (!p || typeof p !== 'string') throw new Error('Path must be a non-empty string'); if (p.includes('\0')) throw new Error('Path contains null bytes'); } async function ensureDir(filePath) { validateMigrationPath(filePath); const dir = dirname(resolve(filePath)); if (!existsSync(dir)) await mkdir(dir, { recursive: true }); } async function atomicWrite(targetPath, data) { validateMigrationPath(targetPath); const abs = resolve(targetPath); const tmp = abs + '.tmp.' + Date.now(); await ensureDir(abs); await writeFile(tmp, data, typeof data === 'string' ? 'utf-8' : undefined); await rename(tmp, abs); } function mkResult(success, entriesMigrated, sourceFormat, targetFormat, startMs, errors) { return { success, entriesMigrated, sourceFormat, targetFormat, durationMs: Date.now() - startMs, errors }; } function normalizeSqliteRow(row) { const out = { ...row }; for (const col of ['tags', 'metadata', 'references']) { if (typeof out[col] === 'string') { try { out[col] = JSON.parse(out[col]); } catch { out[col] = col === 'metadata' ? {} : []; } } } return out; } async function readSqliteRows(dbPath) { // Attempt better-sqlite3 try { const mod = await import('better-sqlite3'); const Database = mod.default ?? mod; const db = new Database(dbPath, { readonly: true }); try { return db.prepare('SELECT * FROM memory_entries').all(); } finally { db.close(); } } catch { /* unavailable */ } // Attempt sql.js try { const mod = await import('sql.js'); const SQL = await (mod.default ?? mod)(); const db = new SQL.Database(await readFile(dbPath)); try { const stmt = db.prepare('SELECT * FROM memory_entries'); const rows = []; while (stmt.step()) rows.push(stmt.getAsObject()); stmt.free(); return rows; } finally { db.close(); } } catch { /* unavailable */ } throw new Error('Cannot read SQLite: install better-sqlite3 or sql.js'); } // -- Batch migration helper ------------------------------------------------- async function migrateBatches(items, rvfPath, options, normalize) { const batchSize = options.batchSize ?? 500; const dimensions = options.dimensions ?? 1536; const backend = new RvfBackend({ databasePath: rvfPath, dimensions, verbose: options.verbose }); await backend.initialize(); let migrated = 0; const errors = []; try { for (let i = 0; i < items.length; i += batchSize) { const batch = items.slice(i, i + batchSize); const entries = []; for (const item of batch) { try { entries.push(fillDefaults(normalize ? normalize(item) : item)); } catch (e) { errors.push(`Entry ${item.id ?? i}: ${e.message}`); } } if (entries.length > 0) { await backend.bulkInsert(entries); migrated += entries.length; } options.onProgress?.({ current: Math.min(i + batchSize, items.length), total: items.length, phase: 'migrating' }); } } finally { await backend.shutdown(); } return { migrated, errors }; } /** * Bidirectional migration utility between RVF and legacy memory formats. * * All methods are static — no instantiation required. */ export class RvfMigrator { /** Migrate a JSON memory file to RVF format. */ static async fromJsonFile(jsonPath, rvfPath, options = {}) { const start = Date.now(); const raw = await readFile(jsonPath, 'utf-8'); let parsed; try { parsed = JSON.parse(raw); } catch (e) { return mkResult(false, 0, 'json', 'rvf', start, [`Invalid JSON: ${e.message}`]); } const items = Array.isArray(parsed) ? parsed : [parsed]; const { migrated, errors } = await migrateBatches(items, rvfPath, options); if (options.verbose) console.log(`[RvfMigrator] Migrated ${migrated} entries from JSON to RVF`); return mkResult(errors.length === 0, migrated, 'json', 'rvf', start, errors); } /** Migrate a SQLite (better-sqlite3 / sql.js) database to RVF. */ static async fromSqlite(dbPath, rvfPath, options = {}) { const start = Date.now(); let rows; try { rows = await readSqliteRows(dbPath); } catch (e) { return mkResult(false, 0, 'sqlite', 'rvf', start, [e.message]); } options.onProgress?.({ current: 0, total: rows.length, phase: 'reading' }); const { migrated, errors } = await migrateBatches(rows, rvfPath, options, normalizeSqliteRow); if (options.verbose) console.log(`[RvfMigrator] Migrated ${migrated} entries from SQLite to RVF`); return mkResult(errors.length === 0, migrated, 'sqlite', 'rvf', start, errors); } /** Export an RVF file back to a JSON array (backward compatibility). */ static async toJsonFile(rvfPath, jsonPath) { const start = Date.now(); const backend = new RvfBackend({ databasePath: rvfPath }); await backend.initialize(); let entries; try { entries = await backend.query({ type: 'hybrid', limit: Number.MAX_SAFE_INTEGER }); } finally { await backend.shutdown(); } const warnings = []; if (entries.length === 0) warnings.push('Source RVF file contained no entries'); await atomicWrite(jsonPath, JSON.stringify(entries.map(serializeForJson), null, 2)); return mkResult(true, entries.length, 'rvf', 'json', start, warnings); } /** * Detect file format by magic bytes. * - RVF\0 (0x52 0x56 0x46 0x00) -> 'rvf' * - SQLi (0x53 0x51 0x4C 0x69) -> 'sqlite' * - Leading [ or { -> 'json' */ static async detectFormat(filePath) { if (!existsSync(filePath)) return 'unknown'; const fd = await import('node:fs').then(m => m.promises.open(filePath, 'r')); try { const buf = Buffer.alloc(16); await fd.read(buf, 0, 16, 0); if (buf[0] === 0x52 && buf[1] === 0x56 && buf[2] === 0x46 && buf[3] === 0x00) return 'rvf'; if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) return 'sqlite'; const head = buf.toString('utf-8').trimStart(); if (head.startsWith('[') || head.startsWith('{')) return 'json'; return 'unknown'; } finally { await fd.close(); } } /** Auto-detect source format and migrate to RVF. */ static async autoMigrate(sourcePath, targetRvfPath, options = {}) { const format = await RvfMigrator.detectFormat(sourcePath); if (options.verbose) console.log(`[RvfMigrator] Detected source format: ${format}`); switch (format) { case 'json': return RvfMigrator.fromJsonFile(sourcePath, targetRvfPath, options); case 'sqlite': return RvfMigrator.fromSqlite(sourcePath, targetRvfPath, options); case 'rvf': return { success: true, entriesMigrated: 0, sourceFormat: 'rvf', targetFormat: 'rvf', durationMs: 0, errors: [] }; default: return { success: false, entriesMigrated: 0, sourceFormat: 'unknown', targetFormat: 'rvf', durationMs: 0, errors: [`Unrecognized format: ${sourcePath}`] }; } } } //# sourceMappingURL=rvf-migration.js.map