Properly send data to storage

This commit is contained in:
Sosthene 2025-09-04 13:19:32 +02:00
parent 7188a33ee8
commit 28db1ae925
4 changed files with 403 additions and 14 deletions

263
package-lock.json generated
View File

@ -1,15 +1,16 @@
{
"name": "sdk_signer",
"version": "1.0.0",
"version": "0.1.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "sdk_signer",
"version": "1.0.0",
"license": "ISC",
"version": "0.1.1",
"license": "MIT",
"dependencies": {
"@types/ws": "^8.5.10",
"axios": "^1.7.8",
"dotenv": "^16.3.1",
"level": "^10.0.0",
"ws": "^8.14.2"
@ -941,6 +942,21 @@
"node": "*"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/axios": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz",
"integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==",
"dependencies": {
"follow-redirects": "^1.15.6",
"form-data": "^4.0.4",
"proxy-from-env": "^1.1.0"
}
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@ -1001,6 +1017,18 @@
"node": ">=8"
}
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/chai": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz",
@ -1048,6 +1076,17 @@
"node": ">=18"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/confbox": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz",
@ -1107,6 +1146,14 @@
"node": ">=6"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@ -1137,6 +1184,60 @@
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-errors": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-object-atoms": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"dependencies": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/esbuild": {
"version": "0.21.5",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz",
@ -1210,6 +1311,40 @@
"url": "https://github.com/sindresorhus/execa?sponsor=1"
}
},
"node_modules/follow-redirects": {
"version": "1.15.11",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
"integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/form-data": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
"integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@ -1225,6 +1360,14 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-func-name": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
@ -1235,6 +1378,41 @@
"node": "*"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/get-stream": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz",
@ -1248,6 +1426,53 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-tostringtag": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"dependencies": {
"has-symbols": "^1.0.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
"dependencies": {
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/human-signals": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz",
@ -1406,6 +1631,14 @@
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
"dev": true
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/maybe-combine-errors": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/maybe-combine-errors/-/maybe-combine-errors-1.0.0.tgz",
@ -1421,6 +1654,25 @@
"dev": true,
"license": "MIT"
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mimic-fn": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz",
@ -1661,6 +1913,11 @@
"node": "^14.15.0 || ^16.10.0 || >=18.0.0"
}
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"node_modules/react-is": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",

View File

@ -20,6 +20,7 @@
"@types/node": "^22.5.0"
},
"dependencies": {
"axios": "^1.7.8",
"ws": "^8.14.2",
"@types/ws": "^8.5.10",
"dotenv": "^16.3.1",

View File

@ -5,6 +5,7 @@ import { ApiReturn, Device, HandshakeMessage, Member, MerkleProofResult, OutPoin
import { RelayManager } from './relay-manager';
import { config } from './config';
import { EMPTY32BYTES } from './utils';
import { storeData } from './storage.service';
const DEFAULTAMOUNT = 1000n;
const DEVICE_KEY = 'main_device';
@ -1026,15 +1027,13 @@ export class Service {
}
}
async saveDataToStorage(hash: string, data: Buffer, ttl: number | null) {
console.log('💾 Saving data to storage:', hash);
console.debug('TODO');
// const storages = [STORAGEURL];
// try {
// await storeData(storages, hash, data, ttl);
// } catch (e) {
// console.error(`Failed to store data with hash ${hash}: ${e}`);
// }
async saveDataToStorage(hash: string, data: Buffer, ttl: number | null, storageUrls: string[]) {
console.log('💾 Saving data', hash, 'to storage', storageUrls);
try {
await storeData(storageUrls, hash, data, ttl);
} catch (e) {
console.error(`Failed to store data with hash ${hash}: ${e}`);
}
}
async saveDiffsToDb(diffs: any[]) {
@ -1052,6 +1051,11 @@ export class Service {
}
}
async getDiffsFromDb(): Promise<Record<string, UserDiff>> {
const db = await Database.getInstance();
return await db.dumpStore('diffs');
}
// Utility methods for data conversion
hexToBlob(hexString: string): Blob {
const uint8Array = this.hexToUInt8Array(hexString);
@ -1162,9 +1166,25 @@ export class Service {
for (const hash of apiReturn.push_to_storage) {
const buffer = await this.getBufferFromDb(hash);
if (buffer) {
await this.saveDataToStorage(hash, buffer, null);
// Look up the storage url for the hash
// Find the field for this hash, then look up the roles to see what storage urls are associated
let storageUrls = new Set<string>();
const diffs = await this.getDiffsFromDb();
const diff = Object.values(diffs).find((diff: UserDiff) => diff.value_commitment === hash);
if (diff) {
for (const role of Object.values(diff.roles)) {
for (const rule of Object.values(role.validation_rules)) {
if (rule.fields.includes(diff.field)) {
for (const storageUrl of role.storages) {
storageUrls.add(storageUrl);
}
}
}
}
}
await this.saveDataToStorage(hash, buffer, null, Array.from(storageUrls));
} else {
console.error('Failed to get data from db');
console.error('Failed to get data from db for hash:', hash);
}
}
}

111
src/storage.service.ts Normal file
View File

@ -0,0 +1,111 @@
import axios, { AxiosResponse } from 'axios';
export async function storeData(servers: string[], key: string, value: Buffer, ttl: number | null): Promise<AxiosResponse | null> {
for (const server of servers) {
try {
// Use key in the URL path instead of query parameters
let url = `${server}/store/${key}`;
// Add ttl as query parameter if provided
if (ttl !== null) {
const urlObj = new URL(url);
urlObj.searchParams.append('ttl', ttl.toString());
url = urlObj.toString();
}
// Send the encrypted ArrayBuffer as the raw request body.
const response = await axios.post(url, value, {
headers: {
'Content-Type': 'application/octet-stream'
},
});
console.log('Data stored successfully:', key);
if (response.status !== 200) {
console.error('Received response status', response.status);
continue;
}
return response;
} catch (error) {
if (axios.isAxiosError(error) && error.response?.status === 409) {
return null;
}
console.error('Error storing data:', error);
}
}
return null;
}
export async function retrieveData(servers: string[], key: string): Promise<ArrayBuffer | null> {
for (const server of servers) {
try {
// Handle relative paths (for development proxy) vs absolute URLs (for production)
const url = server.startsWith('/')
? `${server}/retrieve/${key}` // Relative path - use as-is for proxy
: new URL(`${server}/retrieve/${key}`).toString(); // Absolute URL - construct properly
console.log('Retrieving data', key,' from:', url);
// When fetching the data from the server:
const response = await axios.get(url, {
responseType: 'arraybuffer'
});
if (response.status === 200) {
// Validate that we received an ArrayBuffer
if (response.data instanceof ArrayBuffer) {
return response.data;
} else {
console.error('Server returned non-ArrayBuffer data:', typeof response.data);
continue;
}
} else {
console.error(`Server ${server} returned status ${response.status}`);
continue;
}
} catch (error) {
if (axios.isAxiosError(error)) {
if (error.response?.status === 404) {
console.log(`Data not found on server ${server} for key ${key}`);
continue; // Try next server
} else if (error.response?.status) {
console.error(`Server ${server} error ${error.response.status}:`, error.response.statusText);
continue;
} else {
console.error(`Network error connecting to ${server}:`, error.message);
continue;
}
} else {
console.error(`Unexpected error retrieving data from ${server}:`, error);
continue;
}
}
}
return null;
}
interface TestResponse {
key: string;
value: boolean;
}
export async function testData(servers: string[], key: string): Promise<Record<string, boolean | null> | null> {
const res: Record<string, boolean | null> = {};
for (const server of servers) {
res[server] = null;
try {
const response = await axios.get(`${server}/test/${key}`);
if (response.status !== 200) {
console.error(`${server}: Test response status: ${response.status}`);
continue;
}
const data: TestResponse = response.data;
res[server] = data.value;
} catch (error) {
console.error('Error retrieving data:', error);
return null;
}
}
return res;
}