From 28db1ae9251a8d1d13b4f2a8fc622ca025f32e9e Mon Sep 17 00:00:00 2001 From: Sosthene Date: Thu, 4 Sep 2025 13:19:32 +0200 Subject: [PATCH] Properly send data to storage --- package-lock.json | 263 ++++++++++++++++++++++++++++++++++++++++- package.json | 1 + src/service.ts | 42 +++++-- src/storage.service.ts | 111 +++++++++++++++++ 4 files changed, 403 insertions(+), 14 deletions(-) create mode 100644 src/storage.service.ts diff --git a/package-lock.json b/package-lock.json index 99bb2f5..b28849e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "sdk_signer", - "version": "1.0.0", + "version": "0.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sdk_signer", - "version": "1.0.0", - "license": "ISC", + "version": "0.1.1", + "license": "MIT", "dependencies": { "@types/ws": "^8.5.10", + "axios": "^1.7.8", "dotenv": "^16.3.1", "level": "^10.0.0", "ws": "^8.14.2" @@ -941,6 +942,21 @@ "node": "*" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", + "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1001,6 +1017,18 @@ "node": ">=8" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/chai": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", @@ -1048,6 +1076,17 @@ "node": ">=18" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/confbox": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", @@ -1107,6 +1146,14 @@ "node": ">=6" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -1137,6 +1184,60 @@ "url": "https://dotenvx.com" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -1210,6 +1311,40 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1225,6 +1360,14 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-func-name": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", @@ -1235,6 +1378,41 @@ "node": "*" } }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", @@ -1248,6 +1426,53 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/human-signals": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", @@ -1406,6 +1631,14 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/maybe-combine-errors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/maybe-combine-errors/-/maybe-combine-errors-1.0.0.tgz", @@ -1421,6 +1654,25 @@ "dev": true, "license": "MIT" }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", @@ -1661,6 +1913,11 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", diff --git a/package.json b/package.json index c1ec23b..02bf662 100755 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@types/node": "^22.5.0" }, "dependencies": { + "axios": "^1.7.8", "ws": "^8.14.2", "@types/ws": "^8.5.10", "dotenv": "^16.3.1", diff --git a/src/service.ts b/src/service.ts index 10ec3aa..4cf7fcd 100644 --- a/src/service.ts +++ b/src/service.ts @@ -5,6 +5,7 @@ import { ApiReturn, Device, HandshakeMessage, Member, MerkleProofResult, OutPoin import { RelayManager } from './relay-manager'; import { config } from './config'; import { EMPTY32BYTES } from './utils'; +import { storeData } from './storage.service'; const DEFAULTAMOUNT = 1000n; const DEVICE_KEY = 'main_device'; @@ -1026,15 +1027,13 @@ export class Service { } } - async saveDataToStorage(hash: string, data: Buffer, ttl: number | null) { - console.log('💾 Saving data to storage:', hash); - console.debug('TODO'); - // const storages = [STORAGEURL]; - // try { - // await storeData(storages, hash, data, ttl); - // } catch (e) { - // console.error(`Failed to store data with hash ${hash}: ${e}`); - // } + async saveDataToStorage(hash: string, data: Buffer, ttl: number | null, storageUrls: string[]) { + console.log('💾 Saving data', hash, 'to storage', storageUrls); + try { + await storeData(storageUrls, hash, data, ttl); + } catch (e) { + console.error(`Failed to store data with hash ${hash}: ${e}`); + } } async saveDiffsToDb(diffs: any[]) { @@ -1052,6 +1051,11 @@ export class Service { } } + async getDiffsFromDb(): Promise> { + const db = await Database.getInstance(); + return await db.dumpStore('diffs'); + } + // Utility methods for data conversion hexToBlob(hexString: string): Blob { const uint8Array = this.hexToUInt8Array(hexString); @@ -1162,9 +1166,25 @@ export class Service { for (const hash of apiReturn.push_to_storage) { const buffer = await this.getBufferFromDb(hash); if (buffer) { - await this.saveDataToStorage(hash, buffer, null); + // Look up the storage url for the hash + // Find the field for this hash, then look up the roles to see what storage urls are associated + let storageUrls = new Set(); + const diffs = await this.getDiffsFromDb(); + const diff = Object.values(diffs).find((diff: UserDiff) => diff.value_commitment === hash); + if (diff) { + for (const role of Object.values(diff.roles)) { + for (const rule of Object.values(role.validation_rules)) { + if (rule.fields.includes(diff.field)) { + for (const storageUrl of role.storages) { + storageUrls.add(storageUrl); + } + } + } + } + } + await this.saveDataToStorage(hash, buffer, null, Array.from(storageUrls)); } else { - console.error('Failed to get data from db'); + console.error('Failed to get data from db for hash:', hash); } } } diff --git a/src/storage.service.ts b/src/storage.service.ts new file mode 100644 index 0000000..3dc0cca --- /dev/null +++ b/src/storage.service.ts @@ -0,0 +1,111 @@ +import axios, { AxiosResponse } from 'axios'; + +export async function storeData(servers: string[], key: string, value: Buffer, ttl: number | null): Promise { + for (const server of servers) { + try { + // Use key in the URL path instead of query parameters + let url = `${server}/store/${key}`; + + // Add ttl as query parameter if provided + if (ttl !== null) { + const urlObj = new URL(url); + urlObj.searchParams.append('ttl', ttl.toString()); + url = urlObj.toString(); + } + + // Send the encrypted ArrayBuffer as the raw request body. + const response = await axios.post(url, value, { + headers: { + 'Content-Type': 'application/octet-stream' + }, + }); + console.log('Data stored successfully:', key); + if (response.status !== 200) { + console.error('Received response status', response.status); + continue; + } + return response; + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 409) { + return null; + } + console.error('Error storing data:', error); + } + } + return null; +} + +export async function retrieveData(servers: string[], key: string): Promise { + for (const server of servers) { + try { + // Handle relative paths (for development proxy) vs absolute URLs (for production) + const url = server.startsWith('/') + ? `${server}/retrieve/${key}` // Relative path - use as-is for proxy + : new URL(`${server}/retrieve/${key}`).toString(); // Absolute URL - construct properly + + console.log('Retrieving data', key,' from:', url); + // When fetching the data from the server: + const response = await axios.get(url, { + responseType: 'arraybuffer' + }); + + if (response.status === 200) { + // Validate that we received an ArrayBuffer + if (response.data instanceof ArrayBuffer) { + return response.data; + } else { + console.error('Server returned non-ArrayBuffer data:', typeof response.data); + continue; + } + } else { + console.error(`Server ${server} returned status ${response.status}`); + continue; + } + } catch (error) { + if (axios.isAxiosError(error)) { + if (error.response?.status === 404) { + console.log(`Data not found on server ${server} for key ${key}`); + continue; // Try next server + } else if (error.response?.status) { + console.error(`Server ${server} error ${error.response.status}:`, error.response.statusText); + continue; + } else { + console.error(`Network error connecting to ${server}:`, error.message); + continue; + } + } else { + console.error(`Unexpected error retrieving data from ${server}:`, error); + continue; + } + } + } + return null; +} + +interface TestResponse { + key: string; + value: boolean; +} + +export async function testData(servers: string[], key: string): Promise | null> { + const res: Record = {}; + for (const server of servers) { + res[server] = null; + try { + const response = await axios.get(`${server}/test/${key}`); + if (response.status !== 200) { + console.error(`${server}: Test response status: ${response.status}`); + continue; + } + + const data: TestResponse = response.data; + + res[server] = data.value; + } catch (error) { + console.error('Error retrieving data:', error); + return null; + } + } + + return res; +}