**Motivations:** - Réduction drastique de la consommation mémoire lors des ancrages - Élimination du chargement de 173k+ UTXOs à chaque requête - Stabilisation de la mémoire système sous charge élevée (50+ ancrages/minute) **Root causes:** - api-anchorage chargeait tous les UTXOs (173k+) via listunspent RPC à chaque ancrage - Filtrage et tri de 173k+ objets en mémoire pour sélectionner un seul UTXO - Croissance mémoire de ~16 MB toutes les 12 secondes avec 50 ancrages/minute - Saturation mémoire système en quelques minutes **Correctifs:** - Création du module database.js pour gérer la base de données SQLite partagée - Remplacement de listunspent RPC par requête SQL directe avec LIMIT 1 - Sélection directe d'un UTXO depuis la DB au lieu de charger/filtrer 173k+ objets - Marquage des UTXOs comme dépensés dans la DB après utilisation - Fermeture propre de la base de données lors de l'arrêt **Evolutions:** - Utilisation de la base de données SQLite partagée avec signet-dashboard - Réduction mémoire de 99.999% (173k+ objets → 1 objet par requête) - Amélioration des performances (requête SQL indexée vs filtrage en mémoire) - Optimisation mémoire de signet-dashboard (chargement UTXOs seulement si nécessaire) - Monitoring de lockedUtxos dans api-anchorage pour détecter les fuites - Nettoyage des intervalles frontend pour éviter les fuites mémoire **Pages affectées:** - api-anchorage/src/database.js (nouveau) - api-anchorage/src/bitcoin-rpc.js - api-anchorage/src/server.js - api-anchorage/package.json - signet-dashboard/src/bitcoin-rpc.js - signet-dashboard/public/app.js - features/optimisation-memoire-applications.md (nouveau) - features/api-anchorage-optimisation-base-donnees.md (nouveau)
190 lines
5.0 KiB
TypeScript
190 lines
5.0 KiB
TypeScript
import {
|
|
getSignatures,
|
|
getKeysInWindow,
|
|
getMessageByHash,
|
|
} from '../utils/relay';
|
|
import type { RelayConfig } from '../types/identity';
|
|
import type { LocalIdentity } from '../types/identity';
|
|
import type { MsgChiffre, MsgSignature, MsgCle } from '../types/message';
|
|
import { GraphResolver } from './graphResolver';
|
|
import { validateDecryptedMessage } from './syncValidate';
|
|
import { updateGraphFromMessage } from './syncUpdateGraph';
|
|
import { tryDecryptWithKeys } from './syncDecrypt';
|
|
import { HashCache } from '../utils/cache';
|
|
import { runSyncLoop, type SyncOneRelayResult } from './syncLoop';
|
|
|
|
/**
|
|
* Service for synchronizing messages from relays.
|
|
* Scan-first flow: fetch keys in window → fetch messages by hash → ECDH decrypt.
|
|
*/
|
|
export class SyncService {
|
|
private readonly relays: RelayConfig[];
|
|
private readonly graphResolver: GraphResolver;
|
|
private readonly hashCache: HashCache;
|
|
private readonly identity: LocalIdentity | null;
|
|
|
|
constructor(
|
|
relays: RelayConfig[],
|
|
graphResolver: GraphResolver,
|
|
identity?: LocalIdentity | null,
|
|
) {
|
|
this.relays = relays;
|
|
this.graphResolver = graphResolver;
|
|
this.hashCache = new HashCache();
|
|
this.identity = identity ?? null;
|
|
}
|
|
|
|
/**
|
|
* Initialize HashCache (load from IndexedDB). Call before sync().
|
|
*/
|
|
async init(): Promise<void> {
|
|
await this.hashCache.init();
|
|
}
|
|
|
|
/**
|
|
* Scan-first sync for one relay: fetch keys in window, group by hash,
|
|
* fetch messages by hash, ECDH decrypt, validate, update graph.
|
|
*/
|
|
private async syncOneRelay(
|
|
endpoint: string,
|
|
start: number,
|
|
end: number,
|
|
_serviceUuid?: string,
|
|
): Promise<SyncOneRelayResult> {
|
|
try {
|
|
const keys = await getKeysInWindow(endpoint, start, end);
|
|
const byHash = new Map<string, MsgCle[]>();
|
|
for (const k of keys) {
|
|
const h = k.hash_message;
|
|
const list = byHash.get(h) ?? [];
|
|
list.push(k);
|
|
byHash.set(h, list);
|
|
}
|
|
const newHashes: string[] = [];
|
|
let messages = 0;
|
|
let decrypted = 0;
|
|
let validated = 0;
|
|
let indechiffrable = 0;
|
|
let nonValide = 0;
|
|
for (const [hash, keyList] of byHash) {
|
|
if (this.hashCache.hasSeen(hash)) {
|
|
continue;
|
|
}
|
|
newHashes.push(hash);
|
|
this.hashCache.markSeen(hash);
|
|
let msg: MsgChiffre;
|
|
try {
|
|
msg = await getMessageByHash(endpoint, hash);
|
|
} catch {
|
|
indechiffrable++;
|
|
continue;
|
|
}
|
|
messages++;
|
|
if (this.identity === null) {
|
|
indechiffrable++;
|
|
continue;
|
|
}
|
|
const dec = await tryDecryptWithKeys(msg, keyList, this.identity);
|
|
if (dec === null) {
|
|
indechiffrable++;
|
|
continue;
|
|
}
|
|
decrypted++;
|
|
const valid = await validateDecryptedMessage(
|
|
dec,
|
|
(h) => this.fetchSignatures(h),
|
|
);
|
|
if (valid) {
|
|
updateGraphFromMessage(dec, this.graphResolver);
|
|
validated++;
|
|
} else {
|
|
nonValide++;
|
|
}
|
|
}
|
|
return {
|
|
ok: true,
|
|
newHashes,
|
|
messages,
|
|
newMessages: newHashes.length,
|
|
decrypted,
|
|
validated,
|
|
indechiffrable,
|
|
nonValide,
|
|
};
|
|
} catch (error) {
|
|
console.error(`Error syncing from ${endpoint}:`, error);
|
|
return this.emptyRelayResult();
|
|
}
|
|
}
|
|
|
|
private emptyRelayResult(): SyncOneRelayResult {
|
|
return {
|
|
ok: false,
|
|
newHashes: [],
|
|
messages: 0,
|
|
newMessages: 0,
|
|
decrypted: 0,
|
|
validated: 0,
|
|
indechiffrable: 0,
|
|
nonValide: 0,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Synchronize messages from all enabled relays.
|
|
* Uses scan-first flow (keys in window → fetch by hash → ECDH decrypt).
|
|
*/
|
|
async sync(
|
|
start: number,
|
|
end: number,
|
|
serviceUuid?: string,
|
|
): Promise<{
|
|
messages: number;
|
|
newMessages: number;
|
|
decrypted: number;
|
|
validated: number;
|
|
indechiffrable: number;
|
|
nonValide: number;
|
|
relayStatus: Array<{ endpoint: string; ok: boolean }>;
|
|
}> {
|
|
const fetchOne = (
|
|
ep: string,
|
|
s: number,
|
|
e: number,
|
|
u?: string,
|
|
): Promise<SyncOneRelayResult> =>
|
|
this.syncOneRelay(ep, s, e, u);
|
|
const { acc, relayStatus, allNewHashes } = await runSyncLoop({
|
|
relays: this.relays,
|
|
fetchOne,
|
|
start,
|
|
end,
|
|
serviceUuid,
|
|
});
|
|
if (allNewHashes.length > 0) {
|
|
await this.hashCache.markSeenBatch(allNewHashes);
|
|
}
|
|
return { ...acc, relayStatus };
|
|
}
|
|
|
|
/**
|
|
* Fetch signatures for a message hash (all enabled relays).
|
|
*/
|
|
async fetchSignatures(hash: string): Promise<MsgSignature[]> {
|
|
const allSignatures: MsgSignature[] = [];
|
|
for (const relay of this.relays) {
|
|
if (!relay.enabled) {
|
|
continue;
|
|
}
|
|
try {
|
|
const sigs = await getSignatures(relay.endpoint, hash);
|
|
allSignatures.push(...sigs);
|
|
} catch (error) {
|
|
console.error(`Error fetching signatures from ${relay.endpoint}:`, error);
|
|
}
|
|
}
|
|
return allSignatures;
|
|
}
|
|
|
|
}
|