254 lines
9.7 KiB
JavaScript
254 lines
9.7 KiB
JavaScript
/**
|
|
* RVF Migration Utility — bidirectional migration between RVF and legacy
|
|
* formats (JSON files, sql.js / better-sqlite3 databases).
|
|
* @module @claude-flow/memory/rvf-migration
|
|
*/
|
|
import { readFile, writeFile, rename, mkdir } from 'node:fs/promises';
|
|
import { existsSync } from 'node:fs';
|
|
import { dirname, resolve } from 'node:path';
|
|
import { RvfBackend } from './rvf-backend.js';
|
|
import { generateMemoryId } from './types.js';
|
|
// -- Internal helpers -------------------------------------------------------
|
|
function fillDefaults(raw) {
|
|
const now = Date.now();
|
|
return {
|
|
id: raw.id ?? generateMemoryId(),
|
|
key: raw.key ?? '',
|
|
content: raw.content ?? '',
|
|
type: raw.type ?? 'semantic',
|
|
namespace: raw.namespace ?? 'default',
|
|
tags: raw.tags ?? [],
|
|
metadata: raw.metadata ?? {},
|
|
ownerId: raw.ownerId,
|
|
accessLevel: raw.accessLevel ?? 'private',
|
|
createdAt: raw.createdAt ?? now,
|
|
updatedAt: raw.updatedAt ?? now,
|
|
expiresAt: raw.expiresAt,
|
|
version: raw.version ?? 1,
|
|
references: raw.references ?? [],
|
|
accessCount: raw.accessCount ?? 0,
|
|
lastAccessedAt: raw.lastAccessedAt ?? now,
|
|
embedding: deserializeEmbedding(raw.embedding),
|
|
};
|
|
}
|
|
function deserializeEmbedding(value) {
|
|
if (!value)
|
|
return undefined;
|
|
if (value instanceof Float32Array)
|
|
return value;
|
|
if (value instanceof Buffer || value instanceof Uint8Array) {
|
|
if (value.byteLength === 0)
|
|
return undefined;
|
|
const out = new Float32Array(value.byteLength / 4);
|
|
const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
|
|
for (let i = 0; i < out.length; i++)
|
|
out[i] = view.getFloat32(i * 4, true);
|
|
return out;
|
|
}
|
|
if (Array.isArray(value))
|
|
return new Float32Array(value);
|
|
return undefined;
|
|
}
|
|
function serializeForJson(entry) {
|
|
return { ...entry, embedding: entry.embedding ? Array.from(entry.embedding) : undefined };
|
|
}
|
|
function validateMigrationPath(p) {
|
|
if (!p || typeof p !== 'string')
|
|
throw new Error('Path must be a non-empty string');
|
|
if (p.includes('\0'))
|
|
throw new Error('Path contains null bytes');
|
|
}
|
|
async function ensureDir(filePath) {
|
|
validateMigrationPath(filePath);
|
|
const dir = dirname(resolve(filePath));
|
|
if (!existsSync(dir))
|
|
await mkdir(dir, { recursive: true });
|
|
}
|
|
async function atomicWrite(targetPath, data) {
|
|
validateMigrationPath(targetPath);
|
|
const abs = resolve(targetPath);
|
|
const tmp = abs + '.tmp.' + Date.now();
|
|
await ensureDir(abs);
|
|
await writeFile(tmp, data, typeof data === 'string' ? 'utf-8' : undefined);
|
|
await rename(tmp, abs);
|
|
}
|
|
function mkResult(success, entriesMigrated, sourceFormat, targetFormat, startMs, errors) {
|
|
return { success, entriesMigrated, sourceFormat, targetFormat, durationMs: Date.now() - startMs, errors };
|
|
}
|
|
function normalizeSqliteRow(row) {
|
|
const out = { ...row };
|
|
for (const col of ['tags', 'metadata', 'references']) {
|
|
if (typeof out[col] === 'string') {
|
|
try {
|
|
out[col] = JSON.parse(out[col]);
|
|
}
|
|
catch {
|
|
out[col] = col === 'metadata' ? {} : [];
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
async function readSqliteRows(dbPath) {
|
|
// Attempt better-sqlite3
|
|
try {
|
|
const mod = await import('better-sqlite3');
|
|
const Database = mod.default ?? mod;
|
|
const db = new Database(dbPath, { readonly: true });
|
|
try {
|
|
return db.prepare('SELECT * FROM memory_entries').all();
|
|
}
|
|
finally {
|
|
db.close();
|
|
}
|
|
}
|
|
catch { /* unavailable */ }
|
|
// Attempt sql.js
|
|
try {
|
|
const mod = await import('sql.js');
|
|
const SQL = await (mod.default ?? mod)();
|
|
const db = new SQL.Database(await readFile(dbPath));
|
|
try {
|
|
const stmt = db.prepare('SELECT * FROM memory_entries');
|
|
const rows = [];
|
|
while (stmt.step())
|
|
rows.push(stmt.getAsObject());
|
|
stmt.free();
|
|
return rows;
|
|
}
|
|
finally {
|
|
db.close();
|
|
}
|
|
}
|
|
catch { /* unavailable */ }
|
|
throw new Error('Cannot read SQLite: install better-sqlite3 or sql.js');
|
|
}
|
|
// -- Batch migration helper -------------------------------------------------
|
|
async function migrateBatches(items, rvfPath, options, normalize) {
|
|
const batchSize = options.batchSize ?? 500;
|
|
const dimensions = options.dimensions ?? 1536;
|
|
const backend = new RvfBackend({ databasePath: rvfPath, dimensions, verbose: options.verbose });
|
|
await backend.initialize();
|
|
let migrated = 0;
|
|
const errors = [];
|
|
try {
|
|
for (let i = 0; i < items.length; i += batchSize) {
|
|
const batch = items.slice(i, i + batchSize);
|
|
const entries = [];
|
|
for (const item of batch) {
|
|
try {
|
|
entries.push(fillDefaults(normalize ? normalize(item) : item));
|
|
}
|
|
catch (e) {
|
|
errors.push(`Entry ${item.id ?? i}: ${e.message}`);
|
|
}
|
|
}
|
|
if (entries.length > 0) {
|
|
await backend.bulkInsert(entries);
|
|
migrated += entries.length;
|
|
}
|
|
options.onProgress?.({ current: Math.min(i + batchSize, items.length), total: items.length, phase: 'migrating' });
|
|
}
|
|
}
|
|
finally {
|
|
await backend.shutdown();
|
|
}
|
|
return { migrated, errors };
|
|
}
|
|
/**
|
|
* Bidirectional migration utility between RVF and legacy memory formats.
|
|
*
|
|
* All methods are static — no instantiation required.
|
|
*/
|
|
export class RvfMigrator {
|
|
/** Migrate a JSON memory file to RVF format. */
|
|
static async fromJsonFile(jsonPath, rvfPath, options = {}) {
|
|
const start = Date.now();
|
|
const raw = await readFile(jsonPath, 'utf-8');
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(raw);
|
|
}
|
|
catch (e) {
|
|
return mkResult(false, 0, 'json', 'rvf', start, [`Invalid JSON: ${e.message}`]);
|
|
}
|
|
const items = Array.isArray(parsed) ? parsed : [parsed];
|
|
const { migrated, errors } = await migrateBatches(items, rvfPath, options);
|
|
if (options.verbose)
|
|
console.log(`[RvfMigrator] Migrated ${migrated} entries from JSON to RVF`);
|
|
return mkResult(errors.length === 0, migrated, 'json', 'rvf', start, errors);
|
|
}
|
|
/** Migrate a SQLite (better-sqlite3 / sql.js) database to RVF. */
|
|
static async fromSqlite(dbPath, rvfPath, options = {}) {
|
|
const start = Date.now();
|
|
let rows;
|
|
try {
|
|
rows = await readSqliteRows(dbPath);
|
|
}
|
|
catch (e) {
|
|
return mkResult(false, 0, 'sqlite', 'rvf', start, [e.message]);
|
|
}
|
|
options.onProgress?.({ current: 0, total: rows.length, phase: 'reading' });
|
|
const { migrated, errors } = await migrateBatches(rows, rvfPath, options, normalizeSqliteRow);
|
|
if (options.verbose)
|
|
console.log(`[RvfMigrator] Migrated ${migrated} entries from SQLite to RVF`);
|
|
return mkResult(errors.length === 0, migrated, 'sqlite', 'rvf', start, errors);
|
|
}
|
|
/** Export an RVF file back to a JSON array (backward compatibility). */
|
|
static async toJsonFile(rvfPath, jsonPath) {
|
|
const start = Date.now();
|
|
const backend = new RvfBackend({ databasePath: rvfPath });
|
|
await backend.initialize();
|
|
let entries;
|
|
try {
|
|
entries = await backend.query({ type: 'hybrid', limit: Number.MAX_SAFE_INTEGER });
|
|
}
|
|
finally {
|
|
await backend.shutdown();
|
|
}
|
|
const warnings = [];
|
|
if (entries.length === 0)
|
|
warnings.push('Source RVF file contained no entries');
|
|
await atomicWrite(jsonPath, JSON.stringify(entries.map(serializeForJson), null, 2));
|
|
return mkResult(true, entries.length, 'rvf', 'json', start, warnings);
|
|
}
|
|
/**
|
|
* Detect file format by magic bytes.
|
|
* - RVF\0 (0x52 0x56 0x46 0x00) -> 'rvf'
|
|
* - SQLi (0x53 0x51 0x4C 0x69) -> 'sqlite'
|
|
* - Leading [ or { -> 'json'
|
|
*/
|
|
static async detectFormat(filePath) {
|
|
if (!existsSync(filePath))
|
|
return 'unknown';
|
|
const fd = await import('node:fs').then(m => m.promises.open(filePath, 'r'));
|
|
try {
|
|
const buf = Buffer.alloc(16);
|
|
await fd.read(buf, 0, 16, 0);
|
|
if (buf[0] === 0x52 && buf[1] === 0x56 && buf[2] === 0x46 && buf[3] === 0x00)
|
|
return 'rvf';
|
|
if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69)
|
|
return 'sqlite';
|
|
const head = buf.toString('utf-8').trimStart();
|
|
if (head.startsWith('[') || head.startsWith('{'))
|
|
return 'json';
|
|
return 'unknown';
|
|
}
|
|
finally {
|
|
await fd.close();
|
|
}
|
|
}
|
|
/** Auto-detect source format and migrate to RVF. */
|
|
static async autoMigrate(sourcePath, targetRvfPath, options = {}) {
|
|
const format = await RvfMigrator.detectFormat(sourcePath);
|
|
if (options.verbose)
|
|
console.log(`[RvfMigrator] Detected source format: ${format}`);
|
|
switch (format) {
|
|
case 'json': return RvfMigrator.fromJsonFile(sourcePath, targetRvfPath, options);
|
|
case 'sqlite': return RvfMigrator.fromSqlite(sourcePath, targetRvfPath, options);
|
|
case 'rvf': return { success: true, entriesMigrated: 0, sourceFormat: 'rvf', targetFormat: 'rvf', durationMs: 0, errors: [] };
|
|
default: return { success: false, entriesMigrated: 0, sourceFormat: 'unknown', targetFormat: 'rvf', durationMs: 0, errors: [`Unrecognized format: ${sourcePath}`] };
|
|
}
|
|
}
|
|
}
|
|
//# sourceMappingURL=rvf-migration.js.map
|