From 516819924027d5a440f34baa608d5929a8e90e8e Mon Sep 17 00:00:00 2001 From: Sosthene Date: Fri, 25 Jul 2025 16:49:03 +0200 Subject: [PATCH] Add Database --- .gitignore | 3 +- package-lock.json | 191 +++++++++++++++++++++++++++ package.json | 3 +- src/database.service.ts | 276 ++++++++++++++++++++++++++++++++++++++++ src/service.ts | 74 ++++++++++- 5 files changed, 543 insertions(+), 4 deletions(-) create mode 100644 src/database.service.ts diff --git a/.gitignore b/.gitignore index 8d55725..445011f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules pkg -dist \ No newline at end of file +dist +data \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 852ca94..f86317d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@types/ws": "^8.5.10", "dotenv": "^16.3.1", + "level": "^10.0.0", "ws": "^8.14.2" }, "devDependencies": { @@ -95,6 +96,22 @@ "@types/node": "*" } }, + "node_modules/abstract-level": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-3.1.0.tgz", + "integrity": "sha512-j2e+TsAxy7Ri+0h7dJqwasymgt0zHBWX4+nMk3XatyuqgHfdstBJ9wsMfbiGwE1O+QovRyPcVAqcViMYdyPaaw==", + "dependencies": { + "buffer": "^6.0.3", + "is-buffer": "^2.0.5", + "level-supports": "^6.2.0", + "level-transcoder": "^1.0.1", + "maybe-combine-errors": "^1.0.0", + "module-error": "^1.0.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -125,6 +142,71 @@ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", "dev": true }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/browser-level": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-3.0.0.tgz", + "integrity": "sha512-kGXtLh29jMwqKaskz5xeDLtCtN1KBz/DbQSqmvH7QdJiyGRC7RAM8PPg6gvUiNMa+wVnaxS9eSmEtP/f5ajOVw==", + "dependencies": { + "abstract-level": "^3.1.0" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/classic-level": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/classic-level/-/classic-level-3.0.0.tgz", + "integrity": "sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==", + "hasInstallScript": true, + "dependencies": { + "abstract-level": "^3.1.0", + "module-error": "^1.0.1", + "napi-macros": "^2.2.2", + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -151,12 +233,121 @@ "url": "https://dotenvx.com" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, + "node_modules/level": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/level/-/level-10.0.0.tgz", + "integrity": "sha512-aZJvdfRr/f0VBbSRF5C81FHON47ZsC2TkGxbBezXpGGXAUEL/s6+GP73nnhAYRSCIqUNsmJjfeOF4lzRDKbUig==", + "dependencies": { + "abstract-level": "^3.1.0", + "browser-level": "^3.0.0", + "classic-level": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/level" + } + }, + "node_modules/level-supports": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-6.2.0.tgz", + "integrity": "sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==", + "engines": { + "node": ">=16" + } + }, + "node_modules/level-transcoder": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-transcoder/-/level-transcoder-1.0.1.tgz", + "integrity": "sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==", + "dependencies": { + "buffer": "^6.0.3", + "module-error": "^1.0.1" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "node_modules/maybe-combine-errors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/maybe-combine-errors/-/maybe-combine-errors-1.0.0.tgz", + "integrity": "sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==", + "engines": { + "node": ">=10" + } + }, + "node_modules/module-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", + "integrity": "sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/napi-macros": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", + "integrity": "sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==" + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", diff --git a/package.json b/package.json index 292d20e..624a009 100755 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "dependencies": { "ws": "^8.14.2", "@types/ws": "^8.5.10", - "dotenv": "^16.3.1" + "dotenv": "^16.3.1", + "level": "^10.0.0" } } diff --git a/src/database.service.ts b/src/database.service.ts new file mode 100644 index 0000000..32ee07b --- /dev/null +++ b/src/database.service.ts @@ -0,0 +1,276 @@ +// LevelDB-based persistent key-value database service +// High performance alternative to JSON files + +import { Level } from 'level'; +import * as path from 'path'; + +interface DatabaseObject { + storeName: string; + object: any; + key: string | null; +} + +interface BatchWriteOperation { + storeName: string; + objects: Array<{ key: string; object: any }>; +} + +export default class Database { + private static instance: Database; + private db!: Level; + private dataDir: string; + private initialized: boolean = false; + + private constructor() { + this.dataDir = path.join(process.cwd(), 'data'); + console.log('🔧 Database service initialized (LevelDB)'); + } + + public static async getInstance(): Promise { + if (!Database.instance) { + Database.instance = new Database(); + await Database.instance.init(); + } + return Database.instance; + } + + private async init(): Promise { + if (this.initialized) return; + + try { + // Initialize LevelDB with performance optimizations + this.db = new Level(this.dataDir, { + valueEncoding: 'json', + maxFileSize: 2 * 1024 * 1024, // 2MB + blockSize: 4096, + cacheSize: 8 * 1024 * 1024 // 8MB cache + }); + + this.initialized = true; + console.log('✅ LevelDB database initialized with persistent storage'); + } catch (error) { + console.error('❌ Failed to initialize LevelDB database:', error); + throw error; + } + } + + // Key encoding: "storeName:key" + private getKey(storeName: string, key: string): string { + return `${storeName}:${key}`; + } + + private parseKey(fullKey: string): { storeName: string; key: string } | null { + const parts = fullKey.split(':', 2); + if (parts.length !== 2) return null; + return { storeName: parts[0], key: parts[1] }; + } + + /** + * Get a single object from a store + * O(log n) operation - only reads specific key + */ + public async getObject(storeName: string, key: string): Promise { + try { + const fullKey = this.getKey(storeName, key); + return await this.db.get(fullKey); + } catch (error) { + if ((error as any).code === 'LEVEL_NOT_FOUND') { + return null; + } + throw error; + } + } + + /** + * Add or update an object in a store + * O(log n) operation - only writes specific key-value pair + */ + public async addObject(operation: DatabaseObject): Promise { + const { storeName, object, key } = operation; + + if (key) { + const fullKey = this.getKey(storeName, key); + await this.db.put(fullKey, object); + } else { + // Auto-generate key if none provided + const autoKey = Date.now().toString() + Math.random().toString(36).substr(2, 9); + const fullKey = this.getKey(storeName, autoKey); + await this.db.put(fullKey, object); + } + } + + /** + * Delete an object from a store + * O(log n) operation - only deletes specific key + */ + public async deleteObject(storeName: string, key: string): Promise { + try { + const fullKey = this.getKey(storeName, key); + await this.db.del(fullKey); + } catch (error) { + if ((error as any).code === 'LEVEL_NOT_FOUND') { + // Key doesn't exist, that's fine + return; + } + throw error; + } + } + + /** + * Get all objects from a store as a record + * Efficient range query - only reads keys in the store + */ + public async dumpStore(storeName: string): Promise> { + const result: Record = {}; + const prefix = `${storeName}:`; + + try { + // Use LevelDB's range queries for efficient store dumping + for await (const [key, value] of this.db.iterator({ + gte: prefix, + lt: prefix + '\xff' // '\xff' is higher than any valid character + })) { + const parsed = this.parseKey(key); + if (parsed && parsed.storeName === storeName) { + result[parsed.key] = value; + } + } + } catch (error) { + console.error(`Failed to dump store ${storeName}:`, error); + } + + return result; + } + + /** + * Clear all objects from a store + * Efficient store clearing using range deletes + */ + public async clearStore(storeName: string): Promise { + const prefix = `${storeName}:`; + const batch = this.db.batch(); + + // Collect all keys in the store + for await (const [key] of this.db.iterator({ + gte: prefix, + lt: prefix + '\xff' + })) { + batch.del(key); + } + + await batch.write(); + } + + /** + * Batch write operations + * Atomic batch operations + */ + public async batchWriting(operation: BatchWriteOperation): Promise { + const { storeName, objects } = operation; + + // Use LevelDB's batch operations for atomic writes + const batch = this.db.batch(); + + for (const { key, object } of objects) { + const fullKey = this.getKey(storeName, key); + batch.put(fullKey, object); + } + + await batch.write(); + } + + /** + * Get all store names (for debugging) + */ + public async getStoreNames(): Promise { + const storeNames = new Set(); + + for await (const [key] of this.db.iterator()) { + const parsed = this.parseKey(key); + if (parsed) { + storeNames.add(parsed.storeName); + } + } + + return Array.from(storeNames); + } + + /** + * Get store size (for debugging) + */ + public async getStoreSize(storeName: string): Promise { + let count = 0; + const prefix = `${storeName}:`; + + for await (const [key] of this.db.iterator({ + gte: prefix, + lt: prefix + '\xff' + })) { + count++; + } + + return count; + } + + /** + * Clear all data (for testing/reset) + */ + public async clearAll(): Promise { + // Close and reopen database to clear all data + await this.db.close(); + + // Remove the data directory + const fs = require('fs'); + if (fs.existsSync(this.dataDir)) { + fs.rmSync(this.dataDir, { recursive: true, force: true }); + } + + // Reinitialize + this.initialized = false; + await this.init(); + } + + /** + * Get data directory path (for debugging) + */ + public getDataDirectory(): string { + return this.dataDir; + } + + /** + * Get database statistics (LevelDB specific) + */ + public async getStats(): Promise { + let totalKeys = 0; + let totalStores = 0; + const storeSizes: Record = {}; + + for await (const [key] of this.db.iterator()) { + totalKeys++; + const parsed = this.parseKey(key); + if (parsed) { + if (!storeSizes[parsed.storeName]) { + storeSizes[parsed.storeName] = 0; + totalStores++; + } + storeSizes[parsed.storeName]++; + } + } + + return { + totalKeys, + totalStores, + storeSizes, + dataDirectory: this.dataDir + }; + } + + /** + * Close database connection (should be called on app shutdown) + */ + public async close(): Promise { + if (this.db) { + await this.db.close(); + } + } +} diff --git a/src/service.ts b/src/service.ts index 141cd2f..4a0a5a6 100644 --- a/src/service.ts +++ b/src/service.ts @@ -1,4 +1,5 @@ // Simple server service with core protocol methods using WASM SDK +import Database from './database.service'; import * as wasm from '../pkg/sdk_client'; import { ApiReturn, Device, HandshakeMessage, Member, MerkleProofResult, OutPointProcessMap, Process, ProcessState, RoleDefinition, SecretsStore, UserDiff } from '../pkg/sdk_client'; @@ -54,6 +55,9 @@ export class Service { // Update our cache this.processes.set(processId, result.updated_process.current_process); + // Save to database + await this.saveProcessToDb(processId, result.updated_process.current_process); + return result; } else { throw new Error('Failed to create update message'); @@ -87,6 +91,9 @@ export class Service { // Update our cache this.processes.set(processId, result.updated_process.current_process); + // Save to database + await this.saveProcessToDb(processId, result.updated_process.current_process); + return result; } else { throw new Error('Failed to validate state'); @@ -121,6 +128,9 @@ export class Service { // Update our cache this.processes.set(process.states[0]?.state_id || 'unknown', result.updated_process.current_process); + // Save to database + await this.saveProcessToDb(result.updated_process.process_id, result.updated_process.current_process); + return result; } else { throw new Error('Failed to update process'); @@ -132,11 +142,67 @@ export class Service { // Utility method: Get Process async getProcess(processId: string): Promise { - return this.processes.get(processId) || null; + // First check in-memory cache + const cachedProcess = this.processes.get(processId); + if (cachedProcess) { + return cachedProcess; + } + + // If not in cache, try to get from database + try { + const db = await Database.getInstance(); + const dbProcess = await db.getObject('processes', processId); + if (dbProcess) { + // Cache it for future use + this.processes.set(processId, dbProcess); + return dbProcess; + } + } catch (error) { + console.error('Error getting process from database:', error); + } + + return null; + } + + // Database method: Save Process + async saveProcessToDb(processId: string, process: any): Promise { + try { + const db = await Database.getInstance(); + await db.addObject({ + storeName: 'processes', + object: process, + key: processId + }); + + // Update in-memory cache + this.processes.set(processId, process); + console.log(`💾 Process ${processId} saved to database`); + } catch (error) { + console.error('Error saving process to database:', error); + throw error; + } + } + + // Database method: Get All Processes + async getAllProcesses(): Promise> { + try { + const db = await Database.getInstance(); + const processes = await db.dumpStore('processes'); + + // Update in-memory cache with all processes + for (const [processId, process] of Object.entries(processes)) { + this.processes.set(processId, process); + } + + return processes; + } catch (error) { + console.error('Error getting all processes from database:', error); + return {}; + } } // Utility method: Create a test process - createTestProcess(processId: string): any { + async createTestProcess(processId: string): Promise { console.log(`🔧 Creating test process: ${processId}`); try { @@ -153,6 +219,10 @@ export class Service { if (result.updated_process) { const process = result.updated_process.current_process; this.processes.set(processId, process); + + // Save to database + await this.saveProcessToDb(processId, process); + console.log(`✅ Test process created: ${processId}`); return process; } else {