410 lines
12 KiB
JavaScript
410 lines
12 KiB
JavaScript
/**
|
|
* DatabaseProvider - Platform-aware database selection
|
|
*
|
|
* Automatically selects best backend:
|
|
* - Linux/macOS: better-sqlite3 (native, fast)
|
|
* - Windows: sql.js (WASM, universal) when native fails
|
|
* - Fallback: JSON file storage
|
|
*
|
|
* @module v3/memory/database-provider
|
|
*/
|
|
import { platform } from 'node:os';
|
|
import { existsSync } from 'node:fs';
|
|
import { SQLiteBackend } from './sqlite-backend.js';
|
|
import { SqlJsBackend } from './sqljs-backend.js';
|
|
/**
|
|
* Detect platform and recommend provider
|
|
*/
|
|
function detectPlatform() {
|
|
const os = platform();
|
|
const isWindows = os === 'win32';
|
|
const isMacOS = os === 'darwin';
|
|
const isLinux = os === 'linux';
|
|
// Recommend better-sqlite3 for Unix-like systems, sql.js for Windows
|
|
const recommendedProvider = isWindows ? 'sql.js' : 'better-sqlite3';
|
|
return {
|
|
os,
|
|
isWindows,
|
|
isMacOS,
|
|
isLinux,
|
|
recommendedProvider,
|
|
};
|
|
}
|
|
/**
|
|
* Test if RVF backend is available (always true — pure-TS fallback)
|
|
*/
|
|
async function testRvf() {
|
|
return true;
|
|
}
|
|
/**
|
|
* Test if better-sqlite3 is available and working
|
|
*/
|
|
async function testBetterSqlite3() {
|
|
try {
|
|
const Database = (await import('better-sqlite3')).default;
|
|
const testDb = new Database(':memory:');
|
|
testDb.close();
|
|
return true;
|
|
}
|
|
catch (error) {
|
|
return false;
|
|
}
|
|
}
|
|
/**
|
|
* Test if sql.js is available and working
|
|
*/
|
|
async function testSqlJs() {
|
|
try {
|
|
const initSqlJs = (await import('sql.js')).default;
|
|
const SQL = await initSqlJs();
|
|
const testDb = new SQL.Database();
|
|
testDb.close();
|
|
return true;
|
|
}
|
|
catch (error) {
|
|
return false;
|
|
}
|
|
}
|
|
/**
|
|
* Select best available provider
|
|
*/
|
|
async function selectProvider(preferred, verbose = false) {
|
|
if (preferred && preferred !== 'auto') {
|
|
if (verbose) {
|
|
console.log(`[DatabaseProvider] Using explicitly specified provider: ${preferred}`);
|
|
}
|
|
return preferred;
|
|
}
|
|
const platformInfo = detectPlatform();
|
|
if (verbose) {
|
|
console.log(`[DatabaseProvider] Platform detected: ${platformInfo.os}`);
|
|
console.log(`[DatabaseProvider] Recommended provider: ${platformInfo.recommendedProvider}`);
|
|
}
|
|
// Try RVF first (always available via pure-TS fallback, native when @ruvector/rvf installed)
|
|
if (await testRvf()) {
|
|
if (verbose) {
|
|
console.log('[DatabaseProvider] RVF backend available');
|
|
}
|
|
return 'rvf';
|
|
}
|
|
// Try recommended provider
|
|
if (platformInfo.recommendedProvider === 'better-sqlite3') {
|
|
if (await testBetterSqlite3()) {
|
|
if (verbose) {
|
|
console.log('[DatabaseProvider] better-sqlite3 available and working');
|
|
}
|
|
return 'better-sqlite3';
|
|
}
|
|
else if (verbose) {
|
|
console.log('[DatabaseProvider] better-sqlite3 not available, trying sql.js');
|
|
}
|
|
}
|
|
// Try sql.js as fallback
|
|
if (await testSqlJs()) {
|
|
if (verbose) {
|
|
console.log('[DatabaseProvider] sql.js available and working');
|
|
}
|
|
return 'sql.js';
|
|
}
|
|
else if (verbose) {
|
|
console.log('[DatabaseProvider] sql.js not available, using JSON fallback');
|
|
}
|
|
// Final fallback to JSON
|
|
return 'json';
|
|
}
|
|
/**
|
|
* Create a database instance with platform-aware provider selection
|
|
*
|
|
* @param path - Database file path (:memory: for in-memory)
|
|
* @param options - Database configuration options
|
|
* @returns Initialized database backend
|
|
*
|
|
* @example
|
|
* ```typescript
|
|
* // Auto-select best provider for platform
|
|
* const db = await createDatabase('./data/memory.db');
|
|
*
|
|
* // Force specific provider
|
|
* const db = await createDatabase('./data/memory.db', {
|
|
* provider: 'sql.js'
|
|
* });
|
|
*
|
|
* // With custom options
|
|
* const db = await createDatabase('./data/memory.db', {
|
|
* verbose: true,
|
|
* optimize: true,
|
|
* autoPersistInterval: 10000
|
|
* });
|
|
* ```
|
|
*/
|
|
export async function createDatabase(path, options = {}) {
|
|
const { provider = 'auto', verbose = false, walMode = true, optimize = true, defaultNamespace = 'default', maxEntries = 1000000, autoPersistInterval = 5000, wasmPath, } = options;
|
|
// Select provider
|
|
const selectedProvider = await selectProvider(provider, verbose);
|
|
if (verbose) {
|
|
console.log(`[DatabaseProvider] Creating database with provider: ${selectedProvider}`);
|
|
console.log(`[DatabaseProvider] Database path: ${path}`);
|
|
}
|
|
let backend;
|
|
switch (selectedProvider) {
|
|
case 'better-sqlite3': {
|
|
const config = {
|
|
databasePath: path,
|
|
walMode,
|
|
optimize,
|
|
defaultNamespace,
|
|
maxEntries,
|
|
verbose,
|
|
};
|
|
backend = new SQLiteBackend(config);
|
|
break;
|
|
}
|
|
case 'sql.js': {
|
|
const config = {
|
|
databasePath: path,
|
|
optimize,
|
|
defaultNamespace,
|
|
maxEntries,
|
|
verbose,
|
|
autoPersistInterval,
|
|
wasmPath,
|
|
};
|
|
backend = new SqlJsBackend(config);
|
|
break;
|
|
}
|
|
case 'rvf': {
|
|
const { RvfBackend } = await import('./rvf-backend.js');
|
|
backend = new RvfBackend({
|
|
databasePath: path.replace(/\.(db|json)$/, '.rvf'),
|
|
dimensions: 1536,
|
|
verbose,
|
|
defaultNamespace,
|
|
autoPersistInterval,
|
|
});
|
|
break;
|
|
}
|
|
case 'json': {
|
|
// Simple JSON file backend (minimal implementation)
|
|
backend = new JsonBackend(path, verbose);
|
|
break;
|
|
}
|
|
default:
|
|
throw new Error(`Unknown database provider: ${selectedProvider}`);
|
|
}
|
|
// Initialize the backend
|
|
await backend.initialize();
|
|
if (verbose) {
|
|
console.log(`[DatabaseProvider] Database initialized successfully`);
|
|
}
|
|
return backend;
|
|
}
|
|
/**
|
|
* Get platform information
|
|
*/
|
|
export function getPlatformInfo() {
|
|
return detectPlatform();
|
|
}
|
|
/**
|
|
* Check which providers are available
|
|
*/
|
|
export async function getAvailableProviders() {
|
|
return {
|
|
rvf: true,
|
|
betterSqlite3: await testBetterSqlite3(),
|
|
sqlJs: await testSqlJs(),
|
|
json: true,
|
|
};
|
|
}
|
|
// ===== JSON Fallback Backend =====
|
|
/**
|
|
* Simple JSON file backend for when no SQLite is available
|
|
*/
|
|
class JsonBackend {
|
|
entries = new Map();
|
|
path;
|
|
verbose;
|
|
initialized = false;
|
|
constructor(path, verbose = false) {
|
|
this.path = path;
|
|
this.verbose = verbose;
|
|
}
|
|
async initialize() {
|
|
if (this.initialized)
|
|
return;
|
|
// Load from file if exists
|
|
if (this.path !== ':memory:' && existsSync(this.path)) {
|
|
try {
|
|
const fs = await import('node:fs/promises');
|
|
const data = await fs.readFile(this.path, 'utf-8');
|
|
const entries = JSON.parse(data);
|
|
for (const entry of entries) {
|
|
// Convert embedding array back to Float32Array
|
|
if (entry.embedding) {
|
|
entry.embedding = new Float32Array(entry.embedding);
|
|
}
|
|
this.entries.set(entry.id, entry);
|
|
}
|
|
if (this.verbose) {
|
|
console.log(`[JsonBackend] Loaded ${this.entries.size} entries from ${this.path}`);
|
|
}
|
|
}
|
|
catch (error) {
|
|
if (this.verbose) {
|
|
console.error('[JsonBackend] Error loading file:', error);
|
|
}
|
|
}
|
|
}
|
|
this.initialized = true;
|
|
}
|
|
async shutdown() {
|
|
await this.persist();
|
|
this.initialized = false;
|
|
}
|
|
async store(entry) {
|
|
this.entries.set(entry.id, entry);
|
|
await this.persist();
|
|
}
|
|
async get(id) {
|
|
return this.entries.get(id) || null;
|
|
}
|
|
async getByKey(namespace, key) {
|
|
for (const entry of this.entries.values()) {
|
|
if (entry.namespace === namespace && entry.key === key) {
|
|
return entry;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
async update(id, updateData) {
|
|
const entry = this.entries.get(id);
|
|
if (!entry)
|
|
return null;
|
|
const updated = { ...entry, ...updateData, updatedAt: Date.now(), version: entry.version + 1 };
|
|
this.entries.set(id, updated);
|
|
await this.persist();
|
|
return updated;
|
|
}
|
|
async delete(id) {
|
|
const result = this.entries.delete(id);
|
|
await this.persist();
|
|
return result;
|
|
}
|
|
async query(query) {
|
|
let results = Array.from(this.entries.values());
|
|
if (query.namespace) {
|
|
results = results.filter((e) => e.namespace === query.namespace);
|
|
}
|
|
if (query.key) {
|
|
results = results.filter((e) => e.key === query.key);
|
|
}
|
|
if (query.tags && query.tags.length > 0) {
|
|
results = results.filter((e) => query.tags.every((tag) => e.tags.includes(tag)));
|
|
}
|
|
return results.slice(0, query.limit);
|
|
}
|
|
async search(embedding, options) {
|
|
// Simple brute-force search
|
|
const results = [];
|
|
for (const entry of this.entries.values()) {
|
|
if (!entry.embedding)
|
|
continue;
|
|
const similarity = this.cosineSimilarity(embedding, entry.embedding);
|
|
if (options.threshold && similarity < options.threshold)
|
|
continue;
|
|
results.push({ entry, score: similarity, distance: 1 - similarity });
|
|
}
|
|
results.sort((a, b) => b.score - a.score);
|
|
return results.slice(0, options.k);
|
|
}
|
|
async bulkInsert(entries) {
|
|
for (const entry of entries) {
|
|
this.entries.set(entry.id, entry);
|
|
}
|
|
await this.persist();
|
|
}
|
|
async bulkDelete(ids) {
|
|
let count = 0;
|
|
for (const id of ids) {
|
|
if (this.entries.delete(id))
|
|
count++;
|
|
}
|
|
await this.persist();
|
|
return count;
|
|
}
|
|
async count(namespace) {
|
|
if (!namespace)
|
|
return this.entries.size;
|
|
let count = 0;
|
|
for (const entry of this.entries.values()) {
|
|
if (entry.namespace === namespace)
|
|
count++;
|
|
}
|
|
return count;
|
|
}
|
|
async listNamespaces() {
|
|
const namespaces = new Set();
|
|
for (const entry of this.entries.values()) {
|
|
namespaces.add(entry.namespace);
|
|
}
|
|
return Array.from(namespaces);
|
|
}
|
|
async clearNamespace(namespace) {
|
|
let count = 0;
|
|
for (const [id, entry] of this.entries.entries()) {
|
|
if (entry.namespace === namespace) {
|
|
this.entries.delete(id);
|
|
count++;
|
|
}
|
|
}
|
|
await this.persist();
|
|
return count;
|
|
}
|
|
async getStats() {
|
|
return {
|
|
totalEntries: this.entries.size,
|
|
entriesByNamespace: {},
|
|
entriesByType: {},
|
|
memoryUsage: 0,
|
|
avgQueryTime: 0,
|
|
avgSearchTime: 0,
|
|
};
|
|
}
|
|
async healthCheck() {
|
|
return {
|
|
status: 'healthy',
|
|
components: {
|
|
storage: { status: 'healthy', latency: 0 },
|
|
index: { status: 'healthy', latency: 0 },
|
|
cache: { status: 'healthy', latency: 0 },
|
|
},
|
|
timestamp: Date.now(),
|
|
issues: [],
|
|
recommendations: ['Consider using SQLite backend for better performance'],
|
|
};
|
|
}
|
|
async persist() {
|
|
if (this.path === ':memory:')
|
|
return;
|
|
const fs = await import('node:fs/promises');
|
|
const entries = Array.from(this.entries.values()).map((e) => ({
|
|
...e,
|
|
// Convert Float32Array to regular array for JSON serialization
|
|
embedding: e.embedding ? Array.from(e.embedding) : undefined,
|
|
}));
|
|
await fs.writeFile(this.path, JSON.stringify(entries, null, 2));
|
|
}
|
|
cosineSimilarity(a, b) {
|
|
let dot = 0;
|
|
let normA = 0;
|
|
let normB = 0;
|
|
for (let i = 0; i < a.length; i++) {
|
|
dot += a[i] * b[i];
|
|
normA += a[i] * a[i];
|
|
normB += b[i] * b[i];
|
|
}
|
|
if (normA === 0 || normB === 0)
|
|
return 0;
|
|
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
}
|
|
}
|
|
//# sourceMappingURL=database-provider.js.map
|