Add Database

This commit is contained in:
Sosthene 2025-07-25 16:49:03 +02:00
parent 11f96b44d6
commit 5168199240
5 changed files with 543 additions and 4 deletions

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
node_modules
pkg
dist
dist
data

191
package-lock.json generated
View File

@ -11,6 +11,7 @@
"dependencies": {
"@types/ws": "^8.5.10",
"dotenv": "^16.3.1",
"level": "^10.0.0",
"ws": "^8.14.2"
},
"devDependencies": {
@ -95,6 +96,22 @@
"@types/node": "*"
}
},
"node_modules/abstract-level": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-3.1.0.tgz",
"integrity": "sha512-j2e+TsAxy7Ri+0h7dJqwasymgt0zHBWX4+nMk3XatyuqgHfdstBJ9wsMfbiGwE1O+QovRyPcVAqcViMYdyPaaw==",
"dependencies": {
"buffer": "^6.0.3",
"is-buffer": "^2.0.5",
"level-supports": "^6.2.0",
"level-transcoder": "^1.0.1",
"maybe-combine-errors": "^1.0.0",
"module-error": "^1.0.1"
},
"engines": {
"node": ">=18"
}
},
"node_modules/acorn": {
"version": "8.15.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
@ -125,6 +142,71 @@
"integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
"dev": true
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
]
},
"node_modules/browser-level": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/browser-level/-/browser-level-3.0.0.tgz",
"integrity": "sha512-kGXtLh29jMwqKaskz5xeDLtCtN1KBz/DbQSqmvH7QdJiyGRC7RAM8PPg6gvUiNMa+wVnaxS9eSmEtP/f5ajOVw==",
"dependencies": {
"abstract-level": "^3.1.0"
}
},
"node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/classic-level": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/classic-level/-/classic-level-3.0.0.tgz",
"integrity": "sha512-yGy8j8LjPbN0Bh3+ygmyYvrmskVita92pD/zCoalfcC9XxZj6iDtZTAnz+ot7GG8p9KLTG+MZ84tSA4AhkgVZQ==",
"hasInstallScript": true,
"dependencies": {
"abstract-level": "^3.1.0",
"module-error": "^1.0.1",
"napi-macros": "^2.2.2",
"node-gyp-build": "^4.3.0"
},
"engines": {
"node": ">=18"
}
},
"node_modules/create-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
@ -151,12 +233,121 @@
"url": "https://dotenvx.com"
}
},
"node_modules/ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
]
},
"node_modules/is-buffer": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
"integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"engines": {
"node": ">=4"
}
},
"node_modules/level": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/level/-/level-10.0.0.tgz",
"integrity": "sha512-aZJvdfRr/f0VBbSRF5C81FHON47ZsC2TkGxbBezXpGGXAUEL/s6+GP73nnhAYRSCIqUNsmJjfeOF4lzRDKbUig==",
"dependencies": {
"abstract-level": "^3.1.0",
"browser-level": "^3.0.0",
"classic-level": "^3.0.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/level"
}
},
"node_modules/level-supports": {
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/level-supports/-/level-supports-6.2.0.tgz",
"integrity": "sha512-QNxVXP0IRnBmMsJIh+sb2kwNCYcKciQZJEt+L1hPCHrKNELllXhvrlClVHXBYZVT+a7aTSM6StgNXdAldoab3w==",
"engines": {
"node": ">=16"
}
},
"node_modules/level-transcoder": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/level-transcoder/-/level-transcoder-1.0.1.tgz",
"integrity": "sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==",
"dependencies": {
"buffer": "^6.0.3",
"module-error": "^1.0.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/make-error": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
"dev": true
},
"node_modules/maybe-combine-errors": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/maybe-combine-errors/-/maybe-combine-errors-1.0.0.tgz",
"integrity": "sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==",
"engines": {
"node": ">=10"
}
},
"node_modules/module-error": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz",
"integrity": "sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==",
"engines": {
"node": ">=10"
}
},
"node_modules/napi-macros": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz",
"integrity": "sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g=="
},
"node_modules/node-gyp-build": {
"version": "4.8.4",
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
"integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==",
"bin": {
"node-gyp-build": "bin.js",
"node-gyp-build-optional": "optional.js",
"node-gyp-build-test": "build-test.js"
}
},
"node_modules/ts-node": {
"version": "10.9.2",
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",

View File

@ -20,6 +20,7 @@
"dependencies": {
"ws": "^8.14.2",
"@types/ws": "^8.5.10",
"dotenv": "^16.3.1"
"dotenv": "^16.3.1",
"level": "^10.0.0"
}
}

276
src/database.service.ts Normal file
View File

@ -0,0 +1,276 @@
// LevelDB-based persistent key-value database service
// High performance alternative to JSON files
import { Level } from 'level';
import * as path from 'path';
interface DatabaseObject {
storeName: string;
object: any;
key: string | null;
}
interface BatchWriteOperation {
storeName: string;
objects: Array<{ key: string; object: any }>;
}
export default class Database {
private static instance: Database;
private db!: Level;
private dataDir: string;
private initialized: boolean = false;
private constructor() {
this.dataDir = path.join(process.cwd(), 'data');
console.log('🔧 Database service initialized (LevelDB)');
}
public static async getInstance(): Promise<Database> {
if (!Database.instance) {
Database.instance = new Database();
await Database.instance.init();
}
return Database.instance;
}
private async init(): Promise<void> {
if (this.initialized) return;
try {
// Initialize LevelDB with performance optimizations
this.db = new Level(this.dataDir, {
valueEncoding: 'json',
maxFileSize: 2 * 1024 * 1024, // 2MB
blockSize: 4096,
cacheSize: 8 * 1024 * 1024 // 8MB cache
});
this.initialized = true;
console.log('✅ LevelDB database initialized with persistent storage');
} catch (error) {
console.error('❌ Failed to initialize LevelDB database:', error);
throw error;
}
}
// Key encoding: "storeName:key"
private getKey(storeName: string, key: string): string {
return `${storeName}:${key}`;
}
private parseKey(fullKey: string): { storeName: string; key: string } | null {
const parts = fullKey.split(':', 2);
if (parts.length !== 2) return null;
return { storeName: parts[0], key: parts[1] };
}
/**
* Get a single object from a store
* O(log n) operation - only reads specific key
*/
public async getObject(storeName: string, key: string): Promise<any | null> {
try {
const fullKey = this.getKey(storeName, key);
return await this.db.get(fullKey);
} catch (error) {
if ((error as any).code === 'LEVEL_NOT_FOUND') {
return null;
}
throw error;
}
}
/**
* Add or update an object in a store
* O(log n) operation - only writes specific key-value pair
*/
public async addObject(operation: DatabaseObject): Promise<void> {
const { storeName, object, key } = operation;
if (key) {
const fullKey = this.getKey(storeName, key);
await this.db.put(fullKey, object);
} else {
// Auto-generate key if none provided
const autoKey = Date.now().toString() + Math.random().toString(36).substr(2, 9);
const fullKey = this.getKey(storeName, autoKey);
await this.db.put(fullKey, object);
}
}
/**
* Delete an object from a store
* O(log n) operation - only deletes specific key
*/
public async deleteObject(storeName: string, key: string): Promise<void> {
try {
const fullKey = this.getKey(storeName, key);
await this.db.del(fullKey);
} catch (error) {
if ((error as any).code === 'LEVEL_NOT_FOUND') {
// Key doesn't exist, that's fine
return;
}
throw error;
}
}
/**
* Get all objects from a store as a record
* Efficient range query - only reads keys in the store
*/
public async dumpStore(storeName: string): Promise<Record<string, any>> {
const result: Record<string, any> = {};
const prefix = `${storeName}:`;
try {
// Use LevelDB's range queries for efficient store dumping
for await (const [key, value] of this.db.iterator({
gte: prefix,
lt: prefix + '\xff' // '\xff' is higher than any valid character
})) {
const parsed = this.parseKey(key);
if (parsed && parsed.storeName === storeName) {
result[parsed.key] = value;
}
}
} catch (error) {
console.error(`Failed to dump store ${storeName}:`, error);
}
return result;
}
/**
* Clear all objects from a store
* Efficient store clearing using range deletes
*/
public async clearStore(storeName: string): Promise<void> {
const prefix = `${storeName}:`;
const batch = this.db.batch();
// Collect all keys in the store
for await (const [key] of this.db.iterator({
gte: prefix,
lt: prefix + '\xff'
})) {
batch.del(key);
}
await batch.write();
}
/**
* Batch write operations
* Atomic batch operations
*/
public async batchWriting(operation: BatchWriteOperation): Promise<void> {
const { storeName, objects } = operation;
// Use LevelDB's batch operations for atomic writes
const batch = this.db.batch();
for (const { key, object } of objects) {
const fullKey = this.getKey(storeName, key);
batch.put(fullKey, object);
}
await batch.write();
}
/**
* Get all store names (for debugging)
*/
public async getStoreNames(): Promise<string[]> {
const storeNames = new Set<string>();
for await (const [key] of this.db.iterator()) {
const parsed = this.parseKey(key);
if (parsed) {
storeNames.add(parsed.storeName);
}
}
return Array.from(storeNames);
}
/**
* Get store size (for debugging)
*/
public async getStoreSize(storeName: string): Promise<number> {
let count = 0;
const prefix = `${storeName}:`;
for await (const [key] of this.db.iterator({
gte: prefix,
lt: prefix + '\xff'
})) {
count++;
}
return count;
}
/**
* Clear all data (for testing/reset)
*/
public async clearAll(): Promise<void> {
// Close and reopen database to clear all data
await this.db.close();
// Remove the data directory
const fs = require('fs');
if (fs.existsSync(this.dataDir)) {
fs.rmSync(this.dataDir, { recursive: true, force: true });
}
// Reinitialize
this.initialized = false;
await this.init();
}
/**
* Get data directory path (for debugging)
*/
public getDataDirectory(): string {
return this.dataDir;
}
/**
* Get database statistics (LevelDB specific)
*/
public async getStats(): Promise<any> {
let totalKeys = 0;
let totalStores = 0;
const storeSizes: Record<string, number> = {};
for await (const [key] of this.db.iterator()) {
totalKeys++;
const parsed = this.parseKey(key);
if (parsed) {
if (!storeSizes[parsed.storeName]) {
storeSizes[parsed.storeName] = 0;
totalStores++;
}
storeSizes[parsed.storeName]++;
}
}
return {
totalKeys,
totalStores,
storeSizes,
dataDirectory: this.dataDir
};
}
/**
* Close database connection (should be called on app shutdown)
*/
public async close(): Promise<void> {
if (this.db) {
await this.db.close();
}
}
}

View File

@ -1,4 +1,5 @@
// Simple server service with core protocol methods using WASM SDK
import Database from './database.service';
import * as wasm from '../pkg/sdk_client';
import { ApiReturn, Device, HandshakeMessage, Member, MerkleProofResult, OutPointProcessMap, Process, ProcessState, RoleDefinition, SecretsStore, UserDiff } from '../pkg/sdk_client';
@ -54,6 +55,9 @@ export class Service {
// Update our cache
this.processes.set(processId, result.updated_process.current_process);
// Save to database
await this.saveProcessToDb(processId, result.updated_process.current_process);
return result;
} else {
throw new Error('Failed to create update message');
@ -87,6 +91,9 @@ export class Service {
// Update our cache
this.processes.set(processId, result.updated_process.current_process);
// Save to database
await this.saveProcessToDb(processId, result.updated_process.current_process);
return result;
} else {
throw new Error('Failed to validate state');
@ -121,6 +128,9 @@ export class Service {
// Update our cache
this.processes.set(process.states[0]?.state_id || 'unknown', result.updated_process.current_process);
// Save to database
await this.saveProcessToDb(result.updated_process.process_id, result.updated_process.current_process);
return result;
} else {
throw new Error('Failed to update process');
@ -132,11 +142,67 @@ export class Service {
// Utility method: Get Process
async getProcess(processId: string): Promise<any | null> {
return this.processes.get(processId) || null;
// First check in-memory cache
const cachedProcess = this.processes.get(processId);
if (cachedProcess) {
return cachedProcess;
}
// If not in cache, try to get from database
try {
const db = await Database.getInstance();
const dbProcess = await db.getObject('processes', processId);
if (dbProcess) {
// Cache it for future use
this.processes.set(processId, dbProcess);
return dbProcess;
}
} catch (error) {
console.error('Error getting process from database:', error);
}
return null;
}
// Database method: Save Process
async saveProcessToDb(processId: string, process: any): Promise<void> {
try {
const db = await Database.getInstance();
await db.addObject({
storeName: 'processes',
object: process,
key: processId
});
// Update in-memory cache
this.processes.set(processId, process);
console.log(`💾 Process ${processId} saved to database`);
} catch (error) {
console.error('Error saving process to database:', error);
throw error;
}
}
// Database method: Get All Processes
async getAllProcesses(): Promise<Record<string, any>> {
try {
const db = await Database.getInstance();
const processes = await db.dumpStore('processes');
// Update in-memory cache with all processes
for (const [processId, process] of Object.entries(processes)) {
this.processes.set(processId, process);
}
return processes;
} catch (error) {
console.error('Error getting all processes from database:', error);
return {};
}
}
// Utility method: Create a test process
createTestProcess(processId: string): any {
async createTestProcess(processId: string): Promise<any> {
console.log(`🔧 Creating test process: ${processId}`);
try {
@ -153,6 +219,10 @@ export class Service {
if (result.updated_process) {
const process = result.updated_process.current_process;
this.processes.set(processId, process);
// Save to database
await this.saveProcessToDb(processId, process);
console.log(`✅ Test process created: ${processId}`);
return process;
} else {